def prepare_for_estimate(self, specification_dict = None, specification_storage=None, specification_table=None, events_for_estimation_storage=None, events_for_estimation_table=None, agents_filter='', compute_variables=[], data_objects={}): specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) projects = None # create agents for estimation if events_for_estimation_storage is not None: projects = Dataset(in_storage = events_for_estimation_storage, in_table_name= events_for_estimation_table, id_name=[], dataset_name='development_project' ) if compute_variables: projects.compute_variables(compute_variables, resources=Resources(data_objects)) # needs to be a primary attribute because of the join method below #projects.add_primary_attribute(estimation_set.get_attribute(location_id_variable), # VariableName(location_id_variable).get_alias()) if agents_filter: values = projects.compute_variables(agents_filter, resources=Resources(data_objects)) index = where(values > 0)[0] projects.subset_by_index(index, flush_attributes_if_not_loaded=False) return (specification, projects)
def test_fully_qualified_variable(self): # this tests an expression consisting of a fully-qualified variable expr = "opus_core.test_agent.income_times_2" storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="test_agents", table_data={"income": array([1, 5, 10]), "id": array([1, 3, 4])}) dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array([2, 10, 20]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_variable") # check that expr is in the cache of known expressions # (normally we shouldn't be accessing this private field, but just this once ...) cache = VariableName._cache self.assert_(expr in cache, msg="did not find expr in cache") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), "opus_core", msg="bad value for package") self.assertEqual(name.get_dataset_name(), "test_agent", msg="bad value for dataset") self.assertEqual(name.get_short_name(), "income_times_2", msg="bad value for shortname") self.assertEqual(name.get_alias(), "income_times_2", msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(["income_times_2"]) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable") # check that the cache uses the variable name with whitespace removed oldsize = len(cache) expr_with_spaces = "opus_core . test_agent. income_times_2 " name2 = VariableName(expr_with_spaces) newsize = len(cache) self.assertEqual(oldsize, newsize, msg="caching error") self.assert_(expr_with_spaces not in cache, msg="caching error") self.assertEqual(expr_with_spaces, name2.get_expression(), msg="caching error") self.assertEqual(name2.get_short_name(), "income_times_2", msg="bad value for shortname")
def test_simple_lag_variable(self): test_data = { 1000:{ 'tests':{ 'id':array([1,2,3]), 'attr1':array([10,20,30]), }, }, 1001:{ 'tests':{ 'id':array([1,2,3]), 'attr1':array([111,222,333]), }, }, } cache_creator = CreateTestAttributeCache() cache_creator.create_attribute_cache_with_data(self._temp_dir, test_data) SimulationState().set_current_time(1001) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=['opus_core'], in_storage=attribute_cache) ds = Dataset(in_storage = attribute_cache, in_table_name = 'tests', id_name = ['id'], dataset_name = 'tests') ds.compute_variables(['opus_core.tests.attr1']) self.assert_(ma.allequal(ds.get_attribute('attr1'), array([111,222,333]))) ds.compute_variables(['opus_core.tests.attr1_lag1']) self.assert_(ma.allequal(ds.get_attribute('attr1_lag1'), array([10,20,30])))
def test_fully_qualified_DDD_SSS_variable(self): # this should use the test variable a_test_SSS_variable_DDD_SSS expr = "opus_core.tests.a_test_squid_variable_42_clam" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10,50,100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_DDD_SSS_variable") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset") self.assertEqual(name.get_short_name(), 'a_test_squid_variable_42_clam', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'a_test_squid_variable_42_clam', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(['a_test_squid_variable_42_clam']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_squid_variable_42_clam")
def _compute_variable_for_one_run(self, run_index, variable, dataset_name, year, dataset_arguments={}): dataset_pool = self._setup_environment(self.cache_set[run_index], year) try: ds = dataset_pool.get_dataset(dataset_name, dataset_arguments=dataset_arguments) except: logger.log_warning('Dataset %s could not be loaded from dataset pool using package order %s' % (dataset_name, self.package_order)) logger.log_warning('Generic dataset will be created.') ds = Dataset(in_table_name=dataset_name, in_storage=dataset_pool.get_storage(), dataset_name=dataset_name, id_name=[]) dataset_pool._add_dataset(dataset_name, ds) ds.compute_variables(variable, dataset_pool=dataset_pool) return ds
def prepare_for_estimate(specification_dict = None, specification_storage=None, specification_table=None, agent_set=None, household_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, households_for_estimation_table=None, join_datasets=False, filter=None, data_objects=None): specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) hh_estimation_set = None if households_for_estimation_table is not None: hh_estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=households_for_estimation_table, id_name=household_set.get_id_name(), dataset_name=household_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if filter: estimation_set.compute_variables(filter, resources=Resources(data_objects)) filter_index = where(estimation_set.get_attribute(filter) > 0)[0] #estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: if hh_estimation_set is not None: household_set.join_by_rows(hh_estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index] else: index = agent_set.get_id_index(estimation_set.get_id_attribute()[filter_index]) else: if agent_set is not None: index = arange(agent_set.size()) else: index = None return (specification, index)
def test_two_expressions(self): # test having two different expressions (to make sure having two autogen'd classes at once is working) expr1 = "2*sqrt(my_variable+10)" expr2 = "3*sqrt(my_variable+10)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="dataset", table_data={"my_variable": array([4, -8, 0.5, 1]), "id": array([1, 2, 3, 4])} ) dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset") result1 = dataset.compute_variables([expr1]) should_be1 = array([7.48331477, 2.82842712, 6.4807407, 6.63324958]) self.assert_(ma.allclose(result1, should_be1, rtol=1e-6), "Error in test_two_expressions") result2 = dataset.compute_variables([expr2]) should_be2 = array([11.22497216, 4.24264068, 9.72111105, 9.94987437]) self.assert_(ma.allclose(result2, should_be2, rtol=1e-6), "Error in test_two_expressions")
def test_aggregate_all(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={'my_variable': array([4,8,0.5,1]), 'id': array([1,2,3,4])}) storage.write_table(table_name='regions', table_data={'id': array([1])}) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='regions', id_name="id", dataset_name="myregion") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myregion', ds2) ds2.compute_variables(["myvar = myregion.aggregate_all(myzone.my_variable)"], dataset_pool=dataset_pool) values = ds2.get_attribute("myvar") should_be = array([13.5]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_all")
def define_submarket(choice_set, submarket_id_expression, compute_variables=[], filter=None): submarket_ids = choice_set.compute_variables("submarket_id=" + submarket_id_expression) unique_submarket_ids = unique(submarket_ids) storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="submarkets", table_data={"submarket_id": unique_submarket_ids}) submarkets = Dataset( in_storage=storage, in_table_name="submarkets", id_name="submarket_id", dataset_name="submarket" ) if len(compute_variables): submarkets.compute_variables(compute_variables) if filter is not None: from numpy import logical_not submarkets.remove_elements(index=where(logical_not(submarkets.compute_variables(filter)))[0]) # submarkets = DatasetSubset(submarkets, index=where(submarkets.compute_variables(filter))[0]) return submarkets
def test_constants(self): # test an expression involving two dataset names, one of which is *_constant expr = "test_agent.age<=opus_constant.young_age" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={ "age":array([30,20,60,80]), "id":array([1,3,4,10]) } ) storage.write_table( table_name='opus_constants', table_data={ "young_age":array([35]), "opus_constant_id":array([1]) } ) dataset_pool = DatasetPool(storage=storage) # Test that the dataset name is correct for expr. It should be test_agent -- opus_constant just holds constants, # and is ignored as far as finding the dataset name for the expression. name = VariableName(expr) autogen = name.get_autogen_class() self.assertEqual(name.get_package_name(), None) self.assertEqual(name.get_dataset_name(), 'test_agent') # make an instance of the class and check the dependencies (it shouldn't depend on urbansim_constant) self.assertEqual(autogen().dependencies(), ['test_agent.age']) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr], dataset_pool=dataset_pool) should_be = array( [True,True,False,False] ) self.assertEqual( ma.allequal( result, should_be), True)
def test_disaggregate_one_level(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id0':arange(7)+1, 'id1':array([1,3,1,2,3,2,1]) } ) storage.write_table(table_name='fazes', table_data={ 'id1':array([1,2,3]), 'id2':array([1,2,1]) } ) storage.write_table(table_name='fazdistr', table_data={ 'my_variable':array([40,50]), 'id2':array([1,2]) } ) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds0.compute_variables(["myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"], dataset_pool=dataset_pool) should_be = array([40, 40, 40, 50, 40,50, 40]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_one_level")
def test_disaggregate_and_multiply(self): # Perform two different disaggregations and multiply the results. This tests using a dataset name in both the # list of intermediates and as the dataset being disaggregated (myfaz in this case). expr = "myzone.disaggregate(myfaz.fazsqft) * myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id0':arange(7)+1, 'id1':array([1,3,1,2,3,2,1]) } ) storage.write_table(table_name='fazes', table_data={ 'id1':array([1,2,3]), 'id2':array([1,2,1]), 'fazsqft':array([10,50,100]) } ) storage.write_table(table_name='fazdistrs', table_data={ 'my_variable':array([40,50]), 'id2':array([1,2]) } ) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds0.compute_variables([expr], dataset_pool=dataset_pool) should_be = array([400, 4000, 400, 2500, 4000, 2500, 400]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_and_multiply")
def test_unary_functions_fully_qualified_name(self): # this tests expressions with unary functions applied to a fully qualified name expr = "sqrt(opus_core.tests.a_test_variable)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])} ) dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([3.16227766, 7.0710678, 10]) self.assertEqual( ma.allclose(result, should_be, rtol=1e-3), True, msg="error in test_unary_functions_fully_qualified_name" ) # check that the access methods for the variable all return the correct values name = VariableName(expr) autogen = name.get_autogen_class() self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable") self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), "tests", msg="bad value for dataset") self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname") self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias") # make an instance of the class and check the dependencies (since the dependent variables # all have fully-qualifed names we don't need to associate a dataset with the variable # for this test) self.assertEqual( autogen().dependencies(), ["opus_core.tests.a_test_variable"], msg="dependencies are incorrect" )
def test_number_of_agents(self): expr = "mygridcell.number_of_agents(myjob)" storage = StorageFactory().get_storage('dict_storage') gridcell_grid_id = array([1, 2, 3]) job_grid_id = array([2, 1, 3, 1]) #specify an array of 4 jobs, 1st job's grid_id = 2 (it's in gridcell 2), etc. storage.write_table(table_name='gridcells', table_data={'gid':gridcell_grid_id}) storage.write_table(table_name='jobs', table_data={'jid':arange(4)+1, 'gid':job_grid_id}) gs = Dataset(in_storage=storage, in_table_name='gridcells', id_name="gid", dataset_name="mygridcell") jobs = Dataset(in_storage=storage, in_table_name='jobs', id_name="jid", dataset_name="myjob") values = gs.compute_variables([expr], resources=Resources({"myjob":jobs, "mygridcell":gs})) should_be = array([2, 1, 1]) self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg = "Error in " + expr) # change gids of jobs (to test if computing dependencies is working) jobs.modify_attribute(name="gid", data=array([1,1,1,1])) values2 = gs.compute_variables([expr], resources=Resources({"myjob":jobs, "mygridcell":gs})) should_be2 = array([4, 0, 0]) self.assert_(ma.allclose(values2, should_be2, rtol=1e-7), msg = "Error in " + expr)
def test_rand(self): # test numpy.random.rand (this exercises 0-argument functions) expr = "numpy.random.rand()" storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="dataset", table_data={"id": array([1, 2])}) dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) self.assert_(result >= 0 and result < 1, "Error in test_rand")
def test_numpy_arange_constant(self): # test another constant - syntactically this looks like a method call, so it exercises that part of the code expr = "numpy.arange(5)" storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="dataset", table_data={"id": array([1, 2])}) dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([0, 1, 2, 3, 4]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant")
def test_numpy_arange_constant2(self): # same as test_numpy_arange_constant, except provide 2 arguments expr = "numpy.arange(2,5)" storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="dataset", table_data={"id": array([1, 2])}) dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([2, 3, 4]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant2")
def test_alias_complex_expression(self): # aliasing a complex expression expr = "x = 2*sqrt(var1+var2)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"var1": array([4,-8,0.5,1]), "var2": array([3,3,7,7]), "id": array([1,2,3,4])} ) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([ 5.29150262, 0.0, 5.47722558, 5.65685425]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_alias_complex_expression") # check that the new var has x as an alias v = VariableName(expr) self.assertEqual(v.get_alias(), 'x', msg="bad value for alias") # check that the alias gives the correct value result2 = dataset.compute_variables(['x']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable")
def test_ones_like(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"attr1": array([5, 10, 0, 3]), "attr2": array([5.0, 0.0, 8.0, 1.1]), "id": arange(4)} ) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") expr1 = 'ones_like(attr1)' result = dataset.compute_variables([expr1]) should_be = array([1, 1, 1, 1]) self.assert_(numpy.allclose(result, array(should_be), rtol=1e-6), "Error in ones_like") self.assert_(result.dtype==should_be.dtype, "Error in ones_like") expr2 = 'ones_like(attr1)' result = dataset.compute_variables([expr2]) should_be = array([1, 1, 1, 1]) self.assert_(numpy.allclose(result, array(should_be), rtol=1e-6), "Error in ones_like") self.assertEqual(result.dtype, should_be.dtype, "Error in ones_like")
def function_tester(self, func, values, should_be, optional_args=None): storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="dataset", table_data={"my_variable": array(values), "id": arange(len(values))}) dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset") if optional_args is None: expr = "%s(my_variable)" % func else: expr = "%s(my_variable, %s)" % (func, optional_args) result = dataset.compute_variables([expr]) self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in " + func)
def skip_test_dataset_qualified_name(self): # this tests expressions with a dataset-qualified name expr = "sqrt(tests.a_test_variable)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])} ) dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([3.16227766, 7.0710678, 10]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True)
def test_condition(self): # test using a condition to return an array of True and False values expr = "opus_core.test_agent.income>4" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="test_agents", table_data={"income": array([1, 5, 10, 3]), "id": array([1, 3, 4, 10])} ) dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array([False, True, True, False]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_condition")
def test_true_false(self): # make sure True and False can be used in an expression expr = "array([True, False, False])" # we're not actually using this dataset in the expression, but expressions are computed # with respect to a dataset ... storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="test_agents", table_data={"income": array([10]), "id": array([1])}) dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array([True, False, False]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_true_false")
def test_simple_lag_variable2(self): test_data = { 1000: {"tests": {"id": array([1, 2, 3, 4]), "attr1": array([10, 20, 30, 40])}}, 1001: {"tests": {"id": array([1, 2, 3, 5]), "attr1": array([111, 222, 333, 555])}}, } cache_creator = CreateTestAttributeCache() cache_creator.create_attribute_cache_with_data(self._temp_dir, test_data) SimulationState().set_current_time(1001) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=["opus_core"], in_storage=attribute_cache) ds = Dataset(in_storage=attribute_cache, in_table_name="tests", id_name=["id"], dataset_name="tests") ds.compute_variables(["opus_core.tests.attr1"]) self.assert_(ma.allequal(ds.get_attribute("attr1"), array([111, 222, 333, 555]))) ds.compute_variables(["opus_core.tests.attr1_lag1"]) self.assert_(ma.allequal(ds.get_attribute("attr1_lag1"), array([10, 20, 30, 555])))
def test_safe_array_divide_with_return_value(self): expr = 'safe_array_divide(numerator, denominator, return_value_if_denominator_is_zero=100)' storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"numerator": array([5, 10, 0, 3]), "denominator": array([5, 0, 8, 1]), "id": arange(4)} ) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([1, 100, 0, 3]) self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in safe_array_divide")
def test_safe_array_divide(self): expr = "safe_array_divide(numerator, denominator)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="dataset", table_data={"numerator": array([5, 10, 0, 3]), "denominator": array([5, 0, 8, 1]), "id": arange(4)}, ) dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([1, 0, 0, 3]) self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in safe_array_divide")
def test_where(self): # test using the numpy where function expr = "where(opus_core.test_agent.income>4, 100, 200)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="test_agents", table_data={"income": array([1, 5, 10, 3]), "id": array([1, 3, 4, 10])} ) dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array([200, 100, 100, 200]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_where")
def prepare_for_estimate(self, specification_dict = None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, index_to_unplace=None, portion_to_unplace=1.0, agent_filter=None, data_objects={}): from opus_core.models.model import get_specification_for_estimation specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if (agent_set is not None) and (index_to_unplace is not None): if self.location_id_string is not None: agent_set.compute_variables(self.location_id_string, resources=Resources(data_objects)) if portion_to_unplace < 1: unplace_size = int(portion_to_unplace*index_to_unplace.size) end_index_to_unplace = sample_noreplace(index_to_unplace, unplace_size) else: end_index_to_unplace = index_to_unplace logger.log_status("Unplace " + str(end_index_to_unplace.size) + " agents.") agent_set.modify_attribute(self.choice_set.get_id_name()[0], -1*ones(end_index_to_unplace.size), end_index_to_unplace) # create agents for estimation if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) if agent_filter is not None: estimation_set.compute_variables(agent_filter, resources=Resources(data_objects)) index = where(estimation_set.get_attribute(agent_filter) > 0)[0] estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size()-estimation_set.size(),agent_set.size()) else: index = agent_set.get_id_index(estimation_set.get_id_attribute()) else: index = arange(agent_set.size()) return (specification, index)
def test_casting(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,1000]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") logger.enable_hidden_error_and_warning_words() # Next line should cause a 'WARNING' to be logged. dataset.compute_variables("opus_core.tests.a_test_variable", resources=Resources({"check_variables":"*"})) logger.disable_hidden_error_and_warning_words() values = dataset.get_attribute("a_test_variable") self.assertEqual(values.dtype.type, int8)
def test_compute_unloads_from_memory(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ 'a_dependent_variable':array([1,5,10]), 'id':array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name='id', dataset_name='tests') SessionConfiguration(in_storage=storage)["flush_variables"] = True dataset.get_attribute("a_dependent_variable") self.assert_("a_dependent_variable" in dataset.get_attributes_in_memory()) dataset.compute_variables("opus_core.tests.a_test_variable") self.assert_("a_dependent_variable" not in dataset.get_attributes_in_memory()) self.assert_("a_test_variable" in dataset.get_attributes_in_memory()) SimulationState().remove_singleton(delete_cache=True)
def test_aggregate_sum_two_levels(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable': array([4, 8, 2, 1, 40, 23, 78, 20, 25]), 'id0': arange(9) + 1, 'id1': array([1, 3, 1, 2, 3, 2, 1, 4, 4]) }) storage.write_table(table_name='fazes', table_data={ 'id1': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 3]) }) storage.write_table(table_name='fazdistrs', table_data={ 'id2': array([1, 2, 3]), 'id3': array([1, 2, 1]) }) storage.write_table(table_name='neighborhoods', table_data={"id3": array([1, 2])}) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") ds3 = Dataset(in_storage=storage, in_table_name='neighborhoods', id_name="id3", dataset_name="myneighborhood") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) dataset_pool._add_dataset('myneighborhood', ds3) values = ds3.compute_variables([ 'myneighborhood.aggregate(10.0*myzone.my_variable, intermediates=[myfaz,myfazdistr], function=sum)' ], dataset_pool=dataset_pool) should_be = array([1770, 240]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum_two_levels")
def test_fully_qualified_DDD_SSS_variable(self): # this should use the test variable a_test_SSS_variable_DDD_SSS expr = "opus_core.tests.a_test_squid_variable_42_clam" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10, 50, 100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_DDD_SSS_variable") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset") self.assertEqual(name.get_short_name(), 'a_test_squid_variable_42_clam', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'a_test_squid_variable_42_clam', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(['a_test_squid_variable_42_clam']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_squid_variable_42_clam")
def test_simple_lag_variable(self): test_data = { 1000: { 'tests': { 'id': array([1, 2, 3]), 'attr1': array([10, 20, 30]), }, }, 1001: { 'tests': { 'id': array([1, 2, 3]), 'attr1': array([111, 222, 333]), }, }, } cache_creator = CreateTestAttributeCache() cache_creator.create_attribute_cache_with_data(self._temp_dir, test_data) SimulationState().set_current_time(1001) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=['opus_core'], in_storage=attribute_cache) ds = Dataset(in_storage=attribute_cache, in_table_name='tests', id_name=['id'], dataset_name='tests') ds.compute_variables(['opus_core.tests.attr1']) self.assert_( ma.allequal(ds.get_attribute('attr1'), array([111, 222, 333]))) ds.compute_variables(['opus_core.tests.attr1_lag1']) self.assert_( ma.allequal(ds.get_attribute('attr1_lag1'), array([10, 20, 30])))
def test_casting(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "a_dependent_variable": array([1, 5, 1000]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") logger.enable_hidden_error_and_warning_words() # Next line should cause a 'WARNING' to be logged. dataset.compute_variables("opus_core.tests.a_test_variable", resources=Resources({"check_variables": "*"})) logger.disable_hidden_error_and_warning_words() values = dataset.get_attribute("a_test_variable") self.assertEqual(values.dtype.type, int8)
def test_casts_attribute(self): expr1 = "persons.astype(float64)" expr2 = "persons.astype(float64)**2" expr3 = "(2*persons).astype(float64)" error_msg = "Error in test_casts_attribute" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "persons":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result1 = dataset.compute_variables([expr1]) self.assertEqual( type(result1[0]), float64, error_msg) self.assert_(ma.allclose(result1, array([1, 5, 10]), rtol=1e-6), error_msg) result2 = dataset.compute_variables([expr2]) self.assertEqual( type(result2[0]), float64, error_msg) self.assert_(ma.allclose(result2, array([1, 25, 100]), rtol=1e-6), error_msg) result3 = dataset.compute_variables([expr3]) self.assertEqual( type(result3[0]), float64, error_msg) self.assert_(ma.allclose(result3, array([2, 10, 20]), rtol=1e-6), error_msg)
def test_numpy_arange_constant(self): # test another constant - syntactically this looks like a method call, so it exercises that part of the code expr = "numpy.arange(5)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='dataset', table_data={"id": array([1, 2])}) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([0, 1, 2, 3, 4]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant")
def test_numpy_arange_constant2(self): # same as test_numpy_arange_constant, except provide 2 arguments expr = "numpy.arange(2,5)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='dataset', table_data={"id": array([1, 2])}) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([2, 3, 4]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant2")
def test_casts_fully_qualified_variable(self): expr1 = "opus_core.test_agent.income_times_10.astype(int32)" expr2 = "opus_core.test_agent.income_times_10.astype(int32)**2" expr3 = "(2*opus_core.test_agent.income_times_10).astype(int32)" error_msg = "Error in test_casts_fully_qualified_variable" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={ "income":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result1 = dataset.compute_variables([expr1]) self.assertEqual( type(result1[0]), int32, error_msg) self.assert_(ma.allclose(result1, array([10, 50, 100]), rtol=1e-6), error_msg) result2 = dataset.compute_variables([expr2]) self.assertEqual( type(result2[0]), int32, error_msg) self.assert_(ma.allclose(result2, array([100, 2500, 10000]), rtol=1e-6), error_msg) result3 = dataset.compute_variables([expr3]) self.assertEqual( type(result3[0]), int32, error_msg) self.assert_(ma.allclose(result3, array([20, 100, 200]), rtol=1e-6), error_msg)
def test_disaggregate_two_levels(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='gridcells', table_data={ 'id': arange(9) + 1, 'id0': array([7, 6, 1, 3, 4, 4, 5, 2, 5]) }) storage.write_table(table_name='zones', table_data={ 'id0': arange(7) + 1, 'id1': array([1, 3, 1, 2, 3, 2, 1]) }) storage.write_table(table_name='fazes', table_data={ 'id1': array([1, 2, 3]), 'id2': array([1, 2, 1]) }) storage.write_table(table_name='fazdistrs', table_data={ 'my_variable': array([40, 50]), 'id2': array([1, 2]) }) ds = Dataset(in_storage=storage, in_table_name='gridcells', id_name="id0", dataset_name="mygridcell") ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('mygridcell', ds) dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds.compute_variables([ "mygridcell.disaggregate(myfazdistr.my_variable, intermediates=[myfaz,myzone])" ], dataset_pool=dataset_pool) should_be = array([40, 50, 40, 40, 50, 50, 40, 40, 40]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_two_levels")
def test_variable_dependencies_tree_with_versioning(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ 'id':array([2,4]), 'a_dependent_variable':array([4,7]), 'a_dependent_variable2':array([10,1]) } ) ds = Dataset(in_storage=storage, in_table_name='tests', id_name='id', dataset_name='tests') ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"]) self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==0) #initially version=0 self.assert_(ds.get_version("a_dependent_variable")==0) self.assert_(ds.get_version("a_dependent_variable2")==0) ds.modify_attribute("a_dependent_variable", array([0,0])) self.assert_(ds.get_version("a_dependent_variable")==1) # version=1 ds.modify_attribute("a_dependent_variable", array([1,1])) self.assert_(ds.get_version("a_dependent_variable")==2) # version=2 ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"]) self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==1) ds.compute_variables(["opus_core.tests.a_test_variable_with_two_dependencies"]) self.assert_(ds.get_version("a_test_variable_with_two_dependencies")==1) # version does not change autogen_variable = "my_var = 3 * opus_core.tests.a_dependent_variable" ds.compute_variables([autogen_variable]) self.assert_(ds.get_version("my_var")==0) ds.compute_variables([autogen_variable]) self.assert_(ds.get_version("my_var")==0)
def skip_test_dataset_qualified_name(self): # this tests expressions with a dataset-qualified name expr = "sqrt(tests.a_test_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([3.16227766, 7.0710678, 10]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True)
def test_compute_unloads_from_memory(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ 'a_dependent_variable': array([1, 5, 10]), 'id': array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name='id', dataset_name='tests') SessionConfiguration(in_storage=storage)["flush_variables"] = True dataset.get_attribute("a_dependent_variable") self.assert_( "a_dependent_variable" in dataset.get_attributes_in_memory()) dataset.compute_variables("opus_core.tests.a_test_variable") self.assert_( "a_dependent_variable" not in dataset.get_attributes_in_memory()) self.assert_("a_test_variable" in dataset.get_attributes_in_memory()) SimulationState().remove_singleton(delete_cache=True)
def test_condition(self): # test using a condition to return an array of True and False values expr = "opus_core.test_agent.income>4" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={ "income":array([1,5,10,3]), "id":array([1,3,4,10]) } ) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array( [False,True,True,False] ) self.assertEqual( ma.allclose( result, should_be, rtol=1e-7), True, msg = "Error in test_condition")
def test_where(self): # test using the numpy where function expr = "where(opus_core.test_agent.income>4, 100, 200)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={ "income":array([1,5,10,3]), "id":array([1,3,4,10]) } ) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array( [200, 100, 100, 200] ) self.assertEqual( ma.allclose( result, should_be, rtol=1e-7), True, msg = "Error in test_where")
def prepare_for_estimate(specification_dict=None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, filter=None, agents_filter=None, data_objects=None): """ filter - alias to agents_filter for backforward compatibility, which is more specific """ if agents_filter is None and filter is not None: agents_filter = filter specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage=agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if agents_filter: filter_condition = estimation_set.compute_variables( agents_filter, resources=Resources(data_objects)) filter_index = where(filter_condition)[0] if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index] else: index = agent_set.get_id_index( estimation_set.get_id_attribute()[filter_index]) else: if agent_set is not None: index = arange(agent_set.size()) else: index = None return (specification, index)
def test_sqrt_constant(self): # test an expression that is constant -- should have no dependencies expr = "sqrt(25)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"id": array([1,2])} ) # we don't actually use anything in the dataset dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) self.assert_(4.99<result and result<5.01, "Error in test_sqrt_constant") # check the dependencies v = VariableName(expr) var = VariableFactory().get_variable(v, dataset) self.assertEqual(var.dependencies(), [], msg="dependencies are incorrect")
def test_safe_array_divide_with_return_value(self): expr = 'safe_array_divide(numerator, denominator, return_value_if_denominator_is_zero=100)' storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='dataset', table_data={ "numerator": array([5, 10, 0, 3]), "denominator": array([5, 0, 8, 1]), "id": arange(4) }) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([1, 100, 0, 3]) self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in safe_array_divide")
def test_expression_1var_2times(self): # test an expression with two occurences of the same variable # (the var should just occur once in dependencies) expr = "var1+sqrt(var1)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"var1": array([4,25,0,1]), "id": array([1,2,3,4])} ) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([ 6, 30, 0, 2]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_expression_2vars") # check the dependencies v = VariableName(expr) var = VariableFactory().get_variable(v, dataset) self.assertEqual(var.dependencies(), ['mydataset.var1'], msg="dependencies are incorrect")
def test_true_false(self): # make sure True and False can be used in an expression expr = "array([True, False, False])" # we're not actually using this dataset in the expression, but expressions are computed # with respect to a dataset ... storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={ "income":array([10]), "id":array([1]) } ) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array( [True, False, False] ) self.assertEqual( ma.allclose( result, should_be, rtol=1e-7), True, msg = "Error in test_true_false")
def test_expression(self): dataset_pool = DatasetPool(package_order=['urbansim'], storage=AttributeCache()) storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={ "id": array([1, 2, 3, 4]), "year": array([1968, 1989, 1750, 0]) # absolute_min_year = 1800 }) ds = Dataset(in_storage=storage, in_table_name='dataset', id_name="id") result = ds.compute_variables([ 'is_correct_year = dataset.year >= urbansim_constant.absolute_min_year' ], dataset_pool=dataset_pool) self.assertEqual(ma.allequal(result, array([1, 1, 0, 0])), True)
def function_tester(self, func, values, should_be, optional_args=None): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='dataset', table_data={ "my_variable": array(values), "id": arange(len(values)) }) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") if optional_args is None: expr = "%s(my_variable)" % func else: expr = "%s(my_variable, %s)" % (func, optional_args) result = dataset.compute_variables([expr]) self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in " + func)
def test_expression_2vars(self): # test an expression with 2 variables expr = "2*sqrt(var1+var2)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"var1": array([4,-8,0.5,1]), "var2": array([3,3,7,7]), "id": array([1,2,3,4])} ) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([ 5.29150262, 0.0, 5.47722558, 5.65685425]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_expression_2vars") # check the dependencies (will depend on two different other variables) v = VariableName(expr) var = VariableFactory().get_variable(v, dataset) # use sets for the equality test, since we don't know in what order the dependencies will be returned self.assertEqual(set(var.dependencies()), set(['mydataset.var1', 'mydataset.var2']), msg="dependencies are incorrect")
def test_attr_power(self): # Attributes and fully-qualified names to a power require separate parse tree patterns, # which are tested in the following two tests. # test attribute to a power expr = "var1**3" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='dataset', table_data={"var1": array([4,-8,0.5,1]), "id": array([1,2,3,4])} ) dataset = Dataset(in_storage=storage, in_table_name='dataset', id_name="id", dataset_name="mydataset") result = dataset.compute_variables([expr]) should_be = array([64, -512, 0.125, 1]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_attr_power") # check the dependencies (trickier for ** because we need a separate attribute tree pattern) v = VariableName(expr) var = VariableFactory().get_variable(v, dataset) self.assertEqual(var.dependencies(), ['mydataset.var1'], msg="dependencies are incorrect")
def test_aggregate_sum(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable':array([4,8,0.5,1]), 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]), } ) storage.write_table(table_name='faz', table_data={"id2":array([1,2])}) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myfaz', ds2) values = ds2.compute_variables(['myfaz.aggregate(myzone.my_variable, function=sum)'], dataset_pool=dataset_pool) should_be = array([4.5, 9]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum")
def test_fully_qualified_name_power(self): # test fully qualified name to a power expr = "opus_core.tests.a_test_variable**2" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,0]), "id":array([1,3]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([100,0]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True, msg="error in test_fully_qualified_name_power") # check the dependencies v = VariableName(expr) var = VariableFactory().get_variable(v, dataset) self.assertEqual(var.dependencies(), ['opus_core.tests.a_test_variable'], msg="dependencies are incorrect")
def test_unary_functions_fully_qualified_name(self): # this tests expressions with unary functions applied to a fully qualified name expr = "sqrt(opus_core.tests.a_test_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([3.16227766, 7.0710678, 10]) self.assertEqual( ma.allclose(result, should_be, rtol=1e-5), True, msg="error in test_unary_functions_fully_qualified_name") # check that the access methods for the variable all return the correct values name = VariableName(expr) autogen = name.get_autogen_class() self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable") self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset") self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname") self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias") # make an instance of the class and check the dependencies (since the dependent variables # all have fully-qualifed names we don't need to associate a dataset with the variable # for this test) self.assertEqual(autogen().dependencies(), ['opus_core.tests.a_test_variable'], msg="dependencies are incorrect")
def test_disaggregate_and_multiply(self): # Perform two different disaggregations and multiply the results. This tests using a dataset name in both the # list of intermediates and as the dataset being disaggregated (myfaz in this case). expr = "myzone.disaggregate(myfaz.fazsqft) * myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id0': arange(7) + 1, 'id1': array([1, 3, 1, 2, 3, 2, 1]) }) storage.write_table(table_name='fazes', table_data={ 'id1': array([1, 2, 3]), 'id2': array([1, 2, 1]), 'fazsqft': array([10, 50, 100]) }) storage.write_table(table_name='fazdistrs', table_data={ 'my_variable': array([40, 50]), 'id2': array([1, 2]) }) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds0.compute_variables([expr], dataset_pool=dataset_pool) should_be = array([400, 4000, 400, 2500, 4000, 2500, 400]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_and_multiply")
def test_alias_attribute_with_modification(self): # this tests an expression consisting of an alias for a primary attribute that is modified expr = "p = persons" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "persons": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") # modify the primary attribute 'persons' new_values = array([3, 0, 100]) dataset.modify_attribute('persons', new_values) # result should have the new values result = dataset.compute_variables([expr]) self.assertEqual(ma.allclose(result, new_values, rtol=1e-7), True, msg="error in test_alias_attribute_with_modification")
def test_aggregate_sum_one_level(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable': array([4, 8, 2, 1, 40, 23, 78]), 'id0': arange(7) + 1, 'id1': array([1, 3, 1, 2, 3, 2, 1]) }) storage.write_table(table_name='fazes', table_data={ 'id1': array([1, 2, 3]), 'id2': array([1, 2, 1]) }) storage.write_table(table_name='fazdistr', table_data={'id2': array([1, 2])}) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds2.compute_variables([ 'myfazdistr.aggregate(myzone.my_variable, intermediates=[myfaz])' ], dataset_pool=dataset_pool) should_be = array([132, 24]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum_one_level")
def test_aggregate_squared(self): # more exercising the SUBPATTERN_NUMBER_OF_AGENTS_WITH_CAST tree pattern storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id':array([1,2]), } ) storage.write_table(table_name='gridcells', table_data={ 'my_variable':array([4,8,0.5,1]), 'grid_id':array([1,2,3,4]), 'zone_id':array([1,2,1,2]), } ) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone') gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool() dataset_pool._add_dataset('gridcell', gridcell_dataset) dataset_pool._add_dataset('zone', zone_dataset) values = zone_dataset.compute_variables(['zone.aggregate(gridcell.my_variable)**2'], dataset_pool=dataset_pool) should_be = array([4.5*4.5, 9*9]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate")