def test_interaction_set_number_of_agents_and_multiply2(self): # Similar to test_interaction_set_number_of_agents_and_multiply, except that it uses test_location_x_test_agent expr = "test_agent.income * test_location.number_of_agents(myjob)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={'id':array([1,2,3])} ) storage.write_table( table_name='jobs', table_data={'jid':arange(4)+1, 'id':array([2, 1, 3, 1]) } ) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") jobs = Dataset(in_storage=storage, in_table_name='jobs', id_name="jid", dataset_name="myjob") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myjob', jobs) test_agent_x_test_location = dataset_pool.get_dataset('test_location_x_test_agent') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [2, 1, 1] should_be = array([ [2, 40, 1000], [1, 20, 500], [1, 20, 500] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_interaction_set_number_of_agents_and_multiply2")
def test_interaction_set_number_of_agents(self): # Test number_of_agents on an interaction set component. The interaction set is # test_agent_x_test_location, and we are finding the number of agents in each location expr = "test_location.number_of_agents(myjob)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={'id':array([1,2,3])} ) storage.write_table( table_name='jobs', table_data={'jid':arange(4)+1, 'id':array([2, 1, 3, 1]) } ) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") jobs = Dataset(in_storage=storage, in_table_name='jobs', id_name="jid", dataset_name="myjob") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myjob', jobs) test_agent_x_test_location = dataset_pool.get_dataset('test_agent_x_test_location') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [2, 1, 1] should_be = array([ [2, 1, 1], [2, 1, 1], [2, 1, 1] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_interaction_set_number_of_agents")
def test_interaction_set_aggregate_and_multiply_same_expr(self): # Test doing an aggregate on an interaction set component and using the result in a multiply operation # with the same expression. expr = "(test_agent.income+1)*test_location.aggregate(test_agent.income+1)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ 'id': array([1, 2, 3]), 'location_id': array([1, 2, 2]), 'income': array([1, 20, 50]) }) storage.write_table(table_name='test_locations', table_data={'location_id': array([1, 2])}) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) test_agent_x_test_location = dataset_pool.get_dataset( 'test_agent_x_test_location') result = test_agent_x_test_location.compute_variables( expr, dataset_pool=dataset_pool) # test_agent.income+1 is [2, 21, 51] # test_location.aggregate(test_agent.income+1) is [2, 72] should_be = array([[2 * 2, 2 * 72], [21 * 2, 21 * 72], [51 * 2, 51 * 72]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6))
def test_interaction_set_disaggregate_and_multiply(self): # test doing a disaggregate on an interaction set component and using the result in a multiply operation # this is the same as test_interaction_set_disaggregate except that we multiply by test_agent.income expr = "test_agent.income * test_location.disaggregate(myfaz.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={ 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]) } ) storage.write_table( table_name='faz', table_data={ 'my_variable':array([4,8]), 'id2':array([1,2]) } ) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myfaz', ds2) test_agent_x_test_location = dataset_pool.get_dataset('test_agent_x_test_location') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([ [4, 8, 4, 8], [80, 160, 80, 160], [2000, 4000, 2000, 4000] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate_and_multiply")
def test_my_inputs_convert(self): variable_name = "biocomplexity.land_cover.hmai" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "lct": array([1, 3, 2, 4]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]), 'HEAVY_MED_URBAN': ['HU', 'MU'], 'MU': 2, 'HU': 1, }) land_cover = dataset_pool.get_dataset('land_cover') land_cover.compute_variables(variable_name, dataset_pool=dataset_pool) values = land_cover.get_attribute(variable_name) should_be = arcsin(sqrt(array([0.25, 0, 0.25, 0]))) self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + variable_name)
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "lct": array([11, 2, 4, 3]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "CELLSIZE": 30, "ALL_URBAN": ['HU', 'MU', 'LU'], 'HU': 1, 'MU': 2, 'LU': 3 }) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([1, 0, 1, 0], dtype=float32) should_be = ln(30 * distance_transform_edt(should_be) + 1) / ddt1.standardization_constant_distance self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + self.variable_name)
def test_aggregate_sum(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable': array([4, 8, 0.5, 1]), 'id': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 2]), }) storage.write_table(table_name='faz', table_data={"id2": array([1, 2])}) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myfaz', ds2) values = ds2.compute_variables( ['myfaz.aggregate(10.0*myzone.my_variable, function=sum)'], dataset_pool=dataset_pool) should_be = array([45, 90]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum")
def test_interaction_set_aggregate2(self): # Similar to test_interaction_set_aggregate, except that it uses test_location_x_test_agent expr = "test_location.aggregate(10.0*gridcell.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={ 'location_id':array([1,2]) } ) storage.write_table( table_name='gridcells', table_data={ 'my_variable':array([4,8,0.5,1]), 'grid_id':array([1,2,3,4]), 'location_id':array([1,2,1,2]), } ) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) dataset_pool._add_dataset('gridcell', gridcell_dataset) test_agent_x_test_location = dataset_pool.get_dataset('test_location_x_test_agent') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4.5, 9] should_be = array([ [45, 45, 45], [90, 90, 90] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_aggregate2")
def test_my_inputs_tree_test(self): variable_name = "biocomplexity.land_cover.pcc" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "lct": array([12, 8, 8, 15]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]), "CC": 8, "OW": 12 }) land_cover = dataset_pool.get_dataset('land_cover') land_cover.compute_variables(variable_name, dataset_pool=dataset_pool) values = land_cover.get_attribute(variable_name) cc_within_fp = array([2, 3, 3, 2]) ow_within_fp = array([3, 1, 1, 0]) should_be = cc_within_fp / (5.0 - ow_within_fp) should_be = arcsin(sqrt(should_be)) self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + variable_name)
def test_aggregate(self): # test aggregate with no function specified (so defaults to 'sum') storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id': array([1, 2]), }) storage.write_table(table_name='gridcells', table_data={ 'my_variable': array([4, 8, 0.5, 1]), 'grid_id': array([1, 2, 3, 4]), 'zone_id': array([1, 2, 1, 2]), }) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone') gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool() dataset_pool._add_dataset('gridcell', gridcell_dataset) dataset_pool._add_dataset('zone', zone_dataset) values = zone_dataset.compute_variables( ['zone.aggregate(gridcell.my_variable)'], dataset_pool=dataset_pool) should_be = array([4.5, 9]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate")
def test_no_translation(self): variable_name = "biocomplexity.land_cover.xmps" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,3,1,2,3,1,2,3]), 'relative_y': array([1,1,1,2,2,2,3,3,3]), "lct": array([1, 2, 3, 4, 5, 4, 3, 5, 1]), } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0,1,0], [1,1,1], [0,1,0]]), "ALL_URBAN": ['HU', 'MU', 'LU'], 'HU': 1, 'MU': 2, 'LU': 3 } ) land_cover = dataset_pool.get_dataset('land_cover') self.assertRaises(RuntimeError, land_cover.compute_variables, variable_name, dataset_pool=dataset_pool)
def test_interaction_set_aggregate_and_multiply(self): # test doing an aggregate on an interaction set component and using the result in a multiply operation # this is the same as test_interaction_set_aggregate except that we multiply by test_agent.income expr = "test_agent.income * test_location.aggregate(10.0*gridcell.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={ 'location_id':array([1,2]) } ) storage.write_table( table_name='gridcells', table_data={ 'my_variable':array([4,8,0.5,1]), 'grid_id':array([1,2,3,4]), 'location_id':array([1,2,1,2]), } ) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) dataset_pool._add_dataset('gridcell', gridcell_dataset) test_agent_x_test_location = dataset_pool.get_dataset('test_agent_x_test_location') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4.5, 9] should_be = array([ [45, 90], [900, 1800], [22500, 45000] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_interaction_set_aggregate_and_multiply")
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,1,2]), 'relative_y': array([1,1,2,2]), "lct": array([3, 2, 1, 0]), "footprint_size": array([5, 5, 5, 5]) } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) footprint = array([[0,1,0], [1,1,1], [0,1,0]]) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": footprint, } ) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([3,3,3,0]) / array([5, 5, 5, 5], dtype=float32) should_be = arcsin(sqrt(should_be)) self.assert_(ma.allclose( values, should_be, rtol=1e-7), msg = "Error in " + self.variable_name)
def test_my_inputs_for_hmps(self): variable_name = "biocomplexity.land_cover.hmps" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,1,2]), 'relative_y': array([1,1,2,2]), "lct": array([1, 2, 1, 4]), } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0,1,0], [1,1,1], [0,1,0]]), 'HU': 1, } ) land_cover = dataset_pool.get_dataset('land_cover') land_cover.compute_variables(variable_name, dataset_pool=dataset_pool) values = land_cover.get_attribute(variable_name) should_be = array([2, 2, 2, 2], dtype=float32) should_be = ln(should_be + 1) / SSSmps.standardization_constant_MPS self.assert_(ma.allclose( values, should_be, rtol=1e-7), msg = "Error in " + variable_name)
def test_versioning_with_disaggregate(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]) } ) storage.write_table(table_name='faz', table_data={ 'my_variable':array([4,8]), 'id2':array([1,2]) } ) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myfaz', ds2) var = "my_var = myzone.disaggregate(10.0*myfaz.my_variable)" ds.modify_attribute("id2", array([2,1,2,1])) # should have version 1 ds.compute_variables([var], dataset_pool=dataset_pool) self.assert_(ds.get_version("my_var")==0) ds.compute_variables([var], dataset_pool=dataset_pool) self.assert_(ds.get_version("my_var")==0) # version should stay the same, i.e. it should not recompute ds.touch_attribute("id2") # has version 2 ds.compute_variables([var], dataset_pool=dataset_pool) self.assert_(ds.get_version("my_var")==1) # version should be 1, i.e. it should recompute when id changes
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,1,2]), 'relative_y': array([1,1,2,2]), "lct": array([11,2,4,3]) } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "CELLSIZE":30, "ALL_URBAN":['HU', 'MU', 'LU'], 'HU': 1, 'MU': 2, 'LU': 3 } ) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([1, 0, 1, 0], dtype=float32) should_be = ln(30*distance_transform_edt(should_be)+1) / ddt1.standardization_constant_distance self.assert_(ma.allclose( values, should_be, rtol=1e-7), msg = "Error in " + self.variable_name)
def test_interaction_set_aggregate2(self): # Similar to test_interaction_set_aggregate, except that it uses test_location_x_test_agent expr = "test_location.aggregate(gridcell.my_variable)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="test_agents", table_data={"id": array([1, 2, 3]), "income": array([1, 20, 500])} ) storage.write_table(table_name="test_locations", table_data={"location_id": array([1, 2])}) storage.write_table( table_name="gridcells", table_data={ "my_variable": array([4, 8, 0.5, 1]), "grid_id": array([1, 2, 3, 4]), "location_id": array([1, 2, 1, 2]), }, ) location_dataset = Dataset( in_storage=storage, in_table_name="test_locations", id_name="location_id", dataset_name="test_location" ) gridcell_dataset = Dataset( in_storage=storage, in_table_name="gridcells", id_name="grid_id", dataset_name="gridcell" ) dataset_pool = DatasetPool(package_order=["opus_core"], storage=storage) dataset_pool._add_dataset("test_location", location_dataset) dataset_pool._add_dataset("gridcell", gridcell_dataset) test_agent_x_test_location = dataset_pool.get_dataset("test_location_x_test_agent") result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4.5, 9] should_be = array([[4.5, 4.5, 4.5], [9, 9, 9]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_aggregate2")
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,1,2]), 'relative_y': array([1,1,2,2]), "lct": array([-9999, 5, 3, 1]) } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) footprint = array([[0,1,0], [1,1,1], [0,1,0]]) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": footprint, 'AG': 10, } ) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([2, 4, 4, 5]) self.assert_(ma.allequal( values, should_be), msg = "Error in " + self.variable_name)
def test_aggregate_unqualified_name(self): # test aggregate without the dataset provided for the variable being aggregated expr = 'zone.aggregate(my_variable)' # to be correct, should be 'zone.aggregate(gridcell.my_variable)' storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id': array([1, 2]), }) storage.write_table(table_name='gridcells', table_data={ 'my_variable': array([4, 8, 0.5, 1]), 'grid_id': array([1, 2, 3, 4]), 'zone_id': array([1, 2, 1, 2]), }) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone') gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool() dataset_pool._add_dataset('gridcell', gridcell_dataset) dataset_pool._add_dataset('zone', zone_dataset) self.assertRaises(ValueError, zone_dataset.compute_variables, [expr], dataset_pool=dataset_pool)
def test_aggregate_bad_function(self): # the 'function' argument must be a single name -- test this expr = "zone.aggregate(2*gridcell.my_variable, function=3+4)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id': array([1, 2]), }) storage.write_table(table_name='gridcells', table_data={ 'my_variable': array([4, 8, 0.5, 1]), 'grid_id': array([1, 2, 3, 4]), 'zone_id': array([1, 2, 1, 2]), }) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone') gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool() dataset_pool._add_dataset('gridcell', gridcell_dataset) dataset_pool._add_dataset('zone', zone_dataset) self.assertRaises(ValueError, zone_dataset.compute_variables, [expr], dataset_pool=dataset_pool)
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "lct": array([4, 4, 5, 1]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) footprint = array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) dataset_pool._add_dataset('constant', { "FOOTPRINT": footprint, 'AG': 4, }) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([4, 4, 1, 1]) self.assert_(ma.allequal(values, should_be), msg="Error in " + self.variable_name)
def test_my_inputs_convert(self): variable_name = "biocomplexity.land_cover.hmai" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,1,2]), 'relative_y': array([1,1,2,2]), "lct": array([1, 3, 2, 4]) } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0,1,0], [1,1,1], [0,1,0]]), 'HEAVY_MED_URBAN':['HU','MU'], 'MU': 2, 'HU': 1, } ) land_cover = dataset_pool.get_dataset('land_cover') land_cover.compute_variables(variable_name, dataset_pool=dataset_pool) values = land_cover.get_attribute(variable_name) should_be = arcsin(sqrt(array([0.25, 0, 0.25, 0]))) self.assert_(ma.allclose( values, should_be, rtol=1e-7), msg = "Error in " + variable_name)
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "lct": array([10, 10, 4, 3]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) footprint = array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) dataset_pool._add_dataset('constant', { "FOOTPRINT": footprint, 'AG': 10, }) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([[1, 1], [0, 0]]) should_be = correlate(ma.filled(should_be.astype(int32), 0), footprint, mode="reflect") should_be = less_equal((should_be / 5.0), 400) should_be = ln(distance_transform_edt(should_be) + 1) / dag.standardization_constant_distance should_be = ravel(transpose(should_be)) # flatten by id self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + self.variable_name)
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "comm_add4": array([1, 2, 5, 15]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "CELLSIZE": 250 # this results in a 3x3 grid, (750/250)x(750/250) }) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([ 1 * 4 + 2 * 2 + 5 * 2 + 15, 1 * 2 + 2 * 4 + 5 + 15 * 2, 1 * 2 + 2 + 5 * 4 + 15 * 2, 1 + 2 * 2 + 5 * 2 + 15 * 4 ]) should_be = ln(should_be + 1) / 10.0 self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + self.variable_name)
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "land_cover_type_ow_within_footprint": array([3, 2, 1, 0]), "footprint_size": array([5, 4, 5, 5]) }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) footprint = array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) dataset_pool._add_dataset('constant', { "FOOTPRINT": footprint, }) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([3, 2, 1, 0], dtype=float32) / array([5, 4, 5, 5], dtype=float32) should_be = arcsin(sqrt(should_be)) self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + self.variable_name)
def test_interaction_set_aggregate_and_multiply_same_var(self): # Test doing an aggregate on an interaction set component and using the result in a multiply operation # with the same variable. This is similar to test_interaction_set_aggregate except that we multiply # by the same variable. expr = "test_agent.income*test_location.aggregate(test_agent.income)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ 'id': array([1, 2, 3]), 'location_id': array([1, 2, 2]), 'income': array([1, 20, 50]) }) storage.write_table(table_name='test_locations', table_data={'location_id': array([1, 2])}) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) test_agent_x_test_location = dataset_pool.get_dataset( 'test_agent_x_test_location') result = test_agent_x_test_location.compute_variables( expr, dataset_pool=dataset_pool) # test_agent.income is [1, 20, 50] # test_location.aggregate(test_agent.income) is [1, 70] should_be = array([[1, 70], [20, 1400], [50, 3500]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6))
def test_disaggregate(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 2]) }) storage.write_table(table_name='faz', table_data={ 'my_variable': array([4, 8]), 'id2': array([1, 2]) }) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myfaz', ds2) values = ds.compute_variables( ["myzone.disaggregate(myfaz.my_variable)"], dataset_pool=dataset_pool) should_be = array([4, 8, 4, 8]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate")
def test_disaggregate_fully_qualified_variable(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id': array([1, 2, 3, 4]), 'id': array([1, 2, 1, 2]) }) # it would be nicer to call this table 'fazzes' but we want to use the existing test variable storage.write_table(table_name='test_locations', table_data={ 'cost': array([4, 8]), 'id': array([1, 2]) }) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name="zone") test_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name='test_location') dataset_pool = DatasetPool() dataset_pool._add_dataset('zone', zone_dataset) dataset_pool._add_dataset('test_location', test_dataset) values = zone_dataset.compute_variables( ['zone.disaggregate(opus_core.test_location.cost_times_3)'], dataset_pool=dataset_pool) should_be = array([12, 24, 12, 24]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in test_disaggregate_fully_qualified_variable")
def test_my_inputs(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='land_covers', table_data={ 'relative_x': array([1,2,1,2]), 'relative_y': array([1,1,2,2]), "comm_add4": array([1, 2, 5, 15]) } ) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "CELLSIZE": 250 # this results in a 3x3 grid, (750/250)x(750/250) } ) gridcell = dataset_pool.get_dataset('land_cover') gridcell.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = gridcell.get_attribute(self.variable_name) should_be = array([1*4+2*2+5*2+15, 1*2+2*4+5+15*2, 1*2+2+5*4+15*2, 1+2*2+5*2+15*4]) should_be = ln(should_be + 1) / 10.0 self.assert_(ma.allclose( values, should_be, rtol=1e-7), msg = "Error in " + self.variable_name)
def test_aggregate_all_mean(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable': array([4, 8, 10, 1]), 'id': array([1, 2, 3, 4]), }) storage.write_table(table_name='regions', table_data={ "id": array([1]), }) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='regions', id_name="id", dataset_name="myregion") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myregion', ds2) ds2.compute_variables([ "myvar = myregion.aggregate_all(myzone.my_variable, function=mean)" ], dataset_pool=dataset_pool) values = ds2.get_attribute("myvar") should_be = array([5.75]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_all_mean")
def test_aggregate_squared_with_cast(self): # more exercising the SUBPATTERN_NUMBER_OF_AGENTS_WITH_CAST tree pattern storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id': array([1, 2]), }) storage.write_table(table_name='gridcells', table_data={ 'my_variable': array([4, 8, 0.5, 1]), 'grid_id': array([1, 2, 3, 4]), 'zone_id': array([1, 2, 1, 2]), }) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone') gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool() dataset_pool._add_dataset('gridcell', gridcell_dataset) dataset_pool._add_dataset('zone', zone_dataset) values = zone_dataset.compute_variables( ['(zone.aggregate(gridcell.my_variable)**2).astype(float32)'], dataset_pool=dataset_pool) should_be = array([4.5 * 4.5, 9.0 * 9.0]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate")
def test_aggregate_fully_qualified_variable(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'zone_id': array([1, 2]), }) # it would be nicer to call this table 'gridcells' but we want to use the existing test variable storage.write_table(table_name='tests', table_data={ 'a_dependent_variable': array([4, 8, 0.5, 1]), 'id': array([1, 2, 3, 4]), 'zone_id': array([1, 2, 1, 2]), }) zone_dataset = Dataset(in_storage=storage, in_table_name='zones', id_name="zone_id", dataset_name='zone') test_dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name='tests') dataset_pool = DatasetPool() dataset_pool._add_dataset('zone', zone_dataset) dataset_pool._add_dataset('tests', test_dataset) values = zone_dataset.compute_variables( ['zone.aggregate(opus_core.tests.a_test_variable)'], dataset_pool=dataset_pool) should_be = array([45, 90]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in test_aggregate_fully_qualified_variable")
def test_interaction_set_disaggregate(self): # Test doing a disaggregate on an interaction set component. The interaction set is # test_agent_x_test_location, and test_location will be disaggregated from faz. # (It would be nicer if test_location were called zone, but we wanted to use the existing # test interactions set.) expr = "test_location.disaggregate(10.0*myfaz.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={ 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]) } ) storage.write_table( table_name='faz', table_data={ 'my_variable':array([4,8]), 'id2':array([1,2]) } ) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myfaz', ds2) test_agent_x_test_location = dataset_pool.get_dataset('test_agent_x_test_location') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([ [40, 80, 40, 80], [40, 80, 40, 80], [40, 80, 40, 80] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate")
def test_interaction_set_aggregate(self): # Test doing an aggregate on an interaction set component. The interaction set is # test_agent_x_test_location, and test_location will be aggregated from gridcell. # (It would be nicer if test_location were called zone, but we wanted to use the existing # test interactions set.) expr = "test_location.aggregate(10.0*gridcell.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={ 'location_id':array([1,2]) } ) storage.write_table( table_name='gridcells', table_data={ 'my_variable':array([4,8,0.5,1]), 'grid_id':array([1,2,3,4]), 'location_id':array([1,2,1,2]), } ) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) dataset_pool._add_dataset('gridcell', gridcell_dataset) test_agent_x_test_location = dataset_pool.get_dataset('test_agent_x_test_location') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4.5, 9] should_be = array([ [45, 90], [45, 90], [45, 90] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_aggregate")
def test_interaction_set_disaggregate_and_multiply2(self): # Similar to test_interaction_set_disaggregate_and_multiply, except that it uses test_location_x_test_agent expr = "test_location.disaggregate(10.0*myfaz.my_variable) * test_agent.income" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'income': array([1, 20, 500])} ) storage.write_table( table_name='test_locations', table_data={ 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]) } ) storage.write_table( table_name='faz', table_data={ 'my_variable':array([4,8]), 'id2':array([1,2]) } ) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myfaz', ds2) test_agent_x_test_location = dataset_pool.get_dataset('test_location_x_test_agent') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([ [40, 800, 20000], [80, 1600, 40000], [40, 800, 20000], [80, 1600, 40000] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate_and_multiply2")
def test_my_inputs( self ): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='households', table_data={ 'household_id': array([1, 2, 3, 4]), 'income': array([45000, 50000, 75000, 100000]), } ) dataset_pool = DatasetPool(package_order=['urbansim'], storage=storage) dataset_pool._add_dataset('urbansim_constant', MockConstant()) household = dataset_pool.get_dataset('household') household.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = household.get_attribute(self.variable_name) should_be = array( [1, 1, 0, 0] ) self.assert_(ma.allequal(values, should_be,), msg="Error in " + self.variable_name)
def test_my_tree( self ): storage = StorageFactory().get_storage('dict_storage') #declare an array of four gridcells, each with the specified sector ID below storage.write_table( table_name='development_events', table_data={ 'grid_id': array([100,100,101,102]), 'scheduled_year': array([1999,1998,1999,1999]), 'starting_development_type_id': array([1, 3, 1, 3]), 'ending_development_type_id': array([1, 2, 2, 3]), } ) storage.write_table( table_name='development_type_groups', table_data={ 'name': array(["vacant_developable", "developed"]), 'group_id': array([1,2]), } ) dataset_pool = DatasetPool(package_order=['urbansim', 'opus_core'], storage=storage) dataset_pool._add_dataset('development_type', mock_developmenttype()) development_event = dataset_pool.get_dataset('development_event') development_event.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = development_event.get_attribute(self.variable_name) should_be = array( [True, False, True, False] ) self.assert_(ma.allequal( values, should_be), msg = "Error in " + self.variable_name )
def test_interaction_set_aggregate_and_multiply_same_var(self): # Test doing an aggregate on an interaction set component and using the result in a multiply operation # with the same variable. This is similar to test_interaction_set_aggregate except that we multiply # by the same variable. expr = "test_agent.income*test_location.aggregate(test_agent.income)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={'id': array([1, 2, 3]), 'location_id':array([1,2,2]), 'income': array([1, 20, 50])} ) storage.write_table( table_name='test_locations', table_data={ 'location_id':array([1,2]) } ) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) test_agent_x_test_location = dataset_pool.get_dataset('test_agent_x_test_location') result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # test_agent.income is [1, 20, 50] # test_location.aggregate(test_agent.income) is [1, 70] should_be = array([ [1, 70], [20, 1400], [50, 3500] ]) self.assert_(ma.allclose(result, should_be, rtol=1e-6))
def test_my_inputs_for_hmps(self): variable_name = "biocomplexity.land_cover.hmps" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 1, 2]), 'relative_y': array([1, 1, 2, 2]), "lct": array([1, 2, 1, 4]), }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]), 'HU': 1, }) land_cover = dataset_pool.get_dataset('land_cover') land_cover.compute_variables(variable_name, dataset_pool=dataset_pool) values = land_cover.get_attribute(variable_name) should_be = array([2, 2, 2, 2], dtype=float32) should_be = ln(should_be + 1) / SSSmps.standardization_constant_MPS self.assert_(ma.allclose(values, should_be, rtol=1e-7), msg="Error in " + variable_name)
def test_no_translation(self): variable_name = "biocomplexity.land_cover.xmps" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='land_covers', table_data={ 'relative_x': array([1, 2, 3, 1, 2, 3, 1, 2, 3]), 'relative_y': array([1, 1, 1, 2, 2, 2, 3, 3, 3]), "lct": array([1, 2, 3, 4, 5, 4, 3, 5, 1]), }) dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=storage) dataset_pool._add_dataset( 'constant', { "FOOTPRINT": array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]), "ALL_URBAN": ['HU', 'MU', 'LU'], 'HU': 1, 'MU': 2, 'LU': 3 }) land_cover = dataset_pool.get_dataset('land_cover') self.assertRaises(RuntimeError, land_cover.compute_variables, variable_name, dataset_pool=dataset_pool)
def prepare_dataset_pool(self, recent_years): cache_dir = os.path.join(self.urbansim_tmp, 'urbansim_cache') SimulationState().set_cache_directory(cache_dir) storage = StorageFactory().get_storage('dict_storage') dataset_pool = DatasetPool(package_order=['urbansim'], storage=storage) storage.write_table(table_name='gridcells', table_data={ 'grid_id': array([1, 2, 3, 4]), 'industrial_sqft': array([4, 0, 1, 0]), }) gridcell = dataset_pool.get_dataset('gridcell') self._write_dataset_to_cache(gridcell, cache_dir, 1998) dataset_pool.remove_all_datasets() storage.write_table(table_name='gridcells', table_data={ 'grid_id': array([1, 2, 3, 4]), 'industrial_sqft': array([3, 0, 2, 1]), }) gridcell = dataset_pool.get_dataset('gridcell') self._write_dataset_to_cache(gridcell, cache_dir, 1999) dataset_pool.remove_all_datasets() storage.write_table(table_name='gridcells', table_data={ 'grid_id': array([1, 2, 3, 4]), 'industrial_sqft': array([3, 0, 3, 1]), }) gridcell = dataset_pool.get_dataset('gridcell') self._write_dataset_to_cache(gridcell, cache_dir, 2000) dataset_pool.remove_all_datasets() storage.write_table(table_name='urbansim_constants', table_data={ 'recent_years': array([recent_years]), }) SimulationState().set_current_time(2001) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=['urbansim'], in_storage=attribute_cache) dataset_pool = DatasetPool(package_order=['urbansim'], storage=attribute_cache) # Can't write urbansim_constant, so directly add it to the pool. temp_dataset_pool = DatasetPool(package_order=['urbansim'], storage=storage) dataset_pool._add_dataset( 'urbansim_constant', temp_dataset_pool.get_dataset('urbansim_constant')) return dataset_pool
def prepare_dataset_pool(self, recent_years): self.cache_dir = os.path.join(self.urbansim_tmp, 'urbansim_cache') SimulationState().set_cache_directory(self.cache_dir) data = { 1997: { 'grid_id': array([1, 2, 3, 4]), 'is_in_development_type_group_developed': array([0, 0, 0, 0]), }, 1998: { 'grid_id': array([1, 2, 3, 4]), 'is_in_development_type_group_developed': array([0, 1, 0, 0]), }, 1999: { 'grid_id': array([1, 2, 3, 4]), 'is_in_development_type_group_developed': array([0, 0, 1, 0]) }, 2000: { 'grid_id': array([1, 2, 3, 4]), 'is_in_development_type_group_developed': array([1, 1, 1, 0]) }, 2001: { 'grid_id': array([1, 2, 3, 4]), 'is_in_development_type_group_developed': array([1, 1, 1, 0]) + 1 } } self.write_gridcell_data_to_cache(data) storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='urbansim_constants', table_data={ 'recent_years': array([recent_years]), }) SimulationState().set_current_time(2001) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=['urbansim'], in_storage=attribute_cache) dataset_pool = DatasetPool(package_order=['urbansim'], storage=attribute_cache) # Can't write urbansim_constant, so directly add it to the pool. temp_dataset_pool = DatasetPool(package_order=['urbansim'], storage=storage) dataset_pool._add_dataset( 'urbansim_constant', temp_dataset_pool.get_dataset('urbansim_constant')) return dataset_pool
def prepare_dataset_pool(self, recent_years): self.cache_dir = os.path.join(self.urbansim_tmp, 'urbansim_cache') SimulationState().set_cache_directory(self.cache_dir) data = { 1997:{ 'grid_id':array([1,2,3,4]), 'is_in_development_type_group_developed':array([0,0,0,0]), }, 1998:{ 'grid_id':array([1,2,3,4]), 'is_in_development_type_group_developed':array([0,1,0,0]), }, 1999:{ 'grid_id':array([1,2,3,4]), 'is_in_development_type_group_developed':array([0,0,1,0]) }, 2000:{ 'grid_id':array([1,2,3,4]), 'is_in_development_type_group_developed':array([1,1,1,0]) }, 2001:{ 'grid_id':array([1,2,3,4]), 'is_in_development_type_group_developed':array([1,1,1,0])+1 } } self.write_gridcell_data_to_cache(data) storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='urbansim_constants', table_data={ 'recent_years': array([recent_years]), } ) SimulationState().set_current_time(2001) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=['urbansim'], in_storage=attribute_cache) dataset_pool = DatasetPool(package_order=['urbansim'], storage=attribute_cache) # Can't write urbansim_constant, so directly add it to the pool. temp_dataset_pool = DatasetPool(package_order=['urbansim'], storage=storage) dataset_pool._add_dataset('urbansim_constant', temp_dataset_pool.get_dataset('urbansim_constant')) return dataset_pool
def test_aggregate_all(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={'my_variable': array([4,8,0.5,1]), 'id': array([1,2,3,4])}) storage.write_table(table_name='regions', table_data={'id': array([1])}) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='regions', id_name="id", dataset_name="myregion") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myregion', ds2) ds2.compute_variables(["myvar = myregion.aggregate_all(myzone.my_variable)"], dataset_pool=dataset_pool) values = ds2.get_attribute("myvar") should_be = array([13.5]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_all")
def test_my_inputs( self ): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='jobs', table_data={ 'job_id': array([1,2,3,4]), 'sector_id': array([1, 3, 2, 3]), } ) storage.write_table( table_name='employment_sectors', table_data={ 'sector_id': array([1,2]), 'name': array(["basic", "retail"]), } ) storage.write_table( table_name='employment_adhoc_sector_groups', table_data={ 'group_id': array([1,2]), 'name': array(["basic", "retail"]), } ) storage.write_table( table_name='employment_adhoc_sector_group_definitions', table_data={ 'sector_id': array([1,2]), 'group_id': array([1,2]), } ) dataset_pool = DatasetPool(package_order=['urbansim'], storage=storage) dataset_pool._add_dataset('employment_sector', mock_employmentsector()) job = dataset_pool.get_dataset('job') job.compute_variables(self.variable_name, dataset_pool=dataset_pool) values = job.get_attribute(self.variable_name) should_be = array( [True, False, False, False] ) self.assert_(ma.allequal(values, should_be), msg="Error in " + self.variable_name)
def test_aggregate_sum(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable':array([4,8,0.5,1]), 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]), } ) storage.write_table(table_name='faz', table_data={"id2":array([1,2])}) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myfaz', ds2) values = ds2.compute_variables(['myfaz.aggregate(myzone.my_variable, function=sum)'], dataset_pool=dataset_pool) should_be = array([4.5, 9]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum")
def test_interaction_set_disaggregate(self): # Test doing a disaggregate on an interaction set component. The interaction set is # test_agent_x_test_location, and test_location will be disaggregated from faz. # (It would be nicer if test_location were called zone, but we wanted to use the existing # test interactions set.) expr = "test_location.disaggregate(10.0*myfaz.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ 'id': array([1, 2, 3]), 'income': array([1, 20, 500]) }) storage.write_table(table_name='test_locations', table_data={ 'id': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 2]) }) storage.write_table(table_name='faz', table_data={ 'my_variable': array([4, 8]), 'id2': array([1, 2]) }) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myfaz', ds2) test_agent_x_test_location = dataset_pool.get_dataset( 'test_agent_x_test_location') result = test_agent_x_test_location.compute_variables( expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([[40, 80, 40, 80], [40, 80, 40, 80], [40, 80, 40, 80]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate")
def test_interaction_set_number_of_agents2(self): # Similar to test_interaction_set_number_of_agents, except that it uses test_location_x_test_agent expr = "test_location.number_of_agents(myjob)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="test_agents", table_data={"id": array([1, 2, 3]), "income": array([1, 20, 500])} ) storage.write_table(table_name="test_locations", table_data={"id": array([1, 2, 3])}) storage.write_table(table_name="jobs", table_data={"jid": arange(4) + 1, "id": array([2, 1, 3, 1])}) ds = Dataset(in_storage=storage, in_table_name="test_locations", id_name="id", dataset_name="test_location") jobs = Dataset(in_storage=storage, in_table_name="jobs", id_name="jid", dataset_name="myjob") dataset_pool = DatasetPool(package_order=["opus_core"], storage=storage) dataset_pool._add_dataset("test_location", ds) dataset_pool._add_dataset("myjob", jobs) test_agent_x_test_location = dataset_pool.get_dataset("test_location_x_test_agent") result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [2, 1, 1] should_be = array([[2, 2, 2], [1, 1, 1], [1, 1, 1]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_interaction_set_number_of_agents2")
def test_interaction_set_disaggregate_and_multiply(self): # test doing a disaggregate on an interaction set component and using the result in a multiply operation # this is the same as test_interaction_set_disaggregate except that we multiply by test_agent.income expr = "test_agent.income * test_location.disaggregate(myfaz.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ 'id': array([1, 2, 3]), 'income': array([1, 20, 500]) }) storage.write_table(table_name='test_locations', table_data={ 'id': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 2]) }) storage.write_table(table_name='faz', table_data={ 'my_variable': array([4, 8]), 'id2': array([1, 2]) }) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myfaz', ds2) test_agent_x_test_location = dataset_pool.get_dataset( 'test_agent_x_test_location') result = test_agent_x_test_location.compute_variables( expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([[4, 8, 4, 8], [80, 160, 80, 160], [2000, 4000, 2000, 4000]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate_and_multiply")
def test_aggregate_sum_two_levels(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable': array([4, 8, 2, 1, 40, 23, 78, 20, 25]), 'id0': arange(9) + 1, 'id1': array([1, 3, 1, 2, 3, 2, 1, 4, 4]) }) storage.write_table(table_name='fazes', table_data={ 'id1': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 3]) }) storage.write_table(table_name='fazdistrs', table_data={ 'id2': array([1, 2, 3]), 'id3': array([1, 2, 1]) }) storage.write_table(table_name='neighborhoods', table_data={"id3": array([1, 2])}) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") ds3 = Dataset(in_storage=storage, in_table_name='neighborhoods', id_name="id3", dataset_name="myneighborhood") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) dataset_pool._add_dataset('myneighborhood', ds3) values = ds3.compute_variables([ 'myneighborhood.aggregate(myzone.my_variable, intermediates=[myfaz,myfazdistr], function=sum)' ], dataset_pool=dataset_pool) should_be = array([177, 24]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in aggregate_sum_two_levels")
def test_interaction_set_disaggregate2(self): # Similar to test_interaction_set_disaggregate, except that it uses test_location_x_test_agent expr = "test_location.disaggregate(myfaz.my_variable)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="test_agents", table_data={"id": array([1, 2, 3]), "income": array([1, 20, 500])} ) storage.write_table( table_name="test_locations", table_data={"id": array([1, 2, 3, 4]), "id2": array([1, 2, 1, 2])} ) storage.write_table(table_name="faz", table_data={"my_variable": array([4, 8]), "id2": array([1, 2])}) ds = Dataset(in_storage=storage, in_table_name="test_locations", id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name="faz", id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=["opus_core"], storage=storage) dataset_pool._add_dataset("test_location", ds) dataset_pool._add_dataset("myfaz", ds2) test_agent_x_test_location = dataset_pool.get_dataset("test_location_x_test_agent") result = test_agent_x_test_location.compute_variables(expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([[4, 4, 4], [8, 8, 8], [4, 4, 4], [8, 8, 8]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate2")
def test_aggregate_two_datasets(self): # test aggregating an expression involving two different datasets (error should be caught in parse) expr = 'myfazdistr.aggregate(myzone.my_variable + myfaz.id2, intermediates=[myfaz])' storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'my_variable':array([4,8,2,1,40,23,78]), 'id0':arange(7)+1, 'id1':array([1,3,1,2,3,2,1]) } ) storage.write_table(table_name='fazes', table_data={ 'id1':array([1,2,3]), 'id2':array([1,2,1]) } ) storage.write_table(table_name='fazdistr', table_data={ 'id2':array([1,2]) } ) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz',ds1) dataset_pool._add_dataset('myfazdistr',ds2) self.assertRaises(ValueError, ds2.compute_variables, [expr], dataset_pool=dataset_pool)
def test_disaggregate_one_level(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id0':arange(7)+1, 'id1':array([1,3,1,2,3,2,1]) } ) storage.write_table(table_name='fazes', table_data={ 'id1':array([1,2,3]), 'id2':array([1,2,1]) } ) storage.write_table(table_name='fazdistr', table_data={ 'my_variable':array([40,50]), 'id2':array([1,2]) } ) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds0.compute_variables(["myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"], dataset_pool=dataset_pool) should_be = array([40, 40, 40, 50, 40,50, 40]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_one_level")
def test_disaggregate_and_multiply(self): # Perform two different disaggregations and multiply the results. This tests using a dataset name in both the # list of intermediates and as the dataset being disaggregated (myfaz in this case). expr = "myzone.disaggregate(myfaz.fazsqft) * myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id0':arange(7)+1, 'id1':array([1,3,1,2,3,2,1]) } ) storage.write_table(table_name='fazes', table_data={ 'id1':array([1,2,3]), 'id2':array([1,2,1]), 'fazsqft':array([10,50,100]) } ) storage.write_table(table_name='fazdistrs', table_data={ 'my_variable':array([40,50]), 'id2':array([1,2]) } ) ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds0.compute_variables([expr], dataset_pool=dataset_pool) should_be = array([400, 4000, 400, 2500, 4000, 2500, 400]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_and_multiply")
def test_interaction_set_aggregate(self): # Test doing an aggregate on an interaction set component. The interaction set is # test_agent_x_test_location, and test_location will be aggregated from gridcell. # (It would be nicer if test_location were called zone, but we wanted to use the existing # test interactions set.) expr = "test_location.aggregate(gridcell.my_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ 'id': array([1, 2, 3]), 'income': array([1, 20, 500]) }) storage.write_table(table_name='test_locations', table_data={'location_id': array([1, 2])}) storage.write_table(table_name='gridcells', table_data={ 'my_variable': array([4, 8, 0.5, 1]), 'grid_id': array([1, 2, 3, 4]), 'location_id': array([1, 2, 1, 2]), }) location_dataset = Dataset(in_storage=storage, in_table_name='test_locations', id_name="location_id", dataset_name="test_location") gridcell_dataset = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name='gridcell') dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', location_dataset) dataset_pool._add_dataset('gridcell', gridcell_dataset) test_agent_x_test_location = dataset_pool.get_dataset( 'test_agent_x_test_location') result = test_agent_x_test_location.compute_variables( expr, dataset_pool=dataset_pool) # result for just test_location would be [4.5, 9] should_be = array([[4.5, 9], [4.5, 9], [4.5, 9]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_aggregate")
def test_interaction_set_disaggregate_and_multiply2(self): # Similar to test_interaction_set_disaggregate_and_multiply, except that it uses test_location_x_test_agent expr = "test_location.disaggregate(myfaz.my_variable) * test_agent.income" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ 'id': array([1, 2, 3]), 'income': array([1, 20, 500]) }) storage.write_table(table_name='test_locations', table_data={ 'id': array([1, 2, 3, 4]), 'id2': array([1, 2, 1, 2]) }) storage.write_table(table_name='faz', table_data={ 'my_variable': array([4, 8]), 'id2': array([1, 2]) }) ds = Dataset(in_storage=storage, in_table_name='test_locations', id_name="id", dataset_name="test_location") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool(package_order=['opus_core'], storage=storage) dataset_pool._add_dataset('test_location', ds) dataset_pool._add_dataset('myfaz', ds2) test_agent_x_test_location = dataset_pool.get_dataset( 'test_location_x_test_agent') result = test_agent_x_test_location.compute_variables( expr, dataset_pool=dataset_pool) # result for just test_location would be [4, 8, 4, 8] should_be = array([[4, 80, 2000], [8, 160, 4000], [4, 80, 2000], [8, 160, 4000]]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in interaction_set_disaggregate_and_multiply2")
def test_disaggregate_two_levels(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='gridcells', table_data={ 'id': arange(9) + 1, 'id0': array([7, 6, 1, 3, 4, 4, 5, 2, 5]) }) storage.write_table(table_name='zones', table_data={ 'id0': arange(7) + 1, 'id1': array([1, 3, 1, 2, 3, 2, 1]) }) storage.write_table(table_name='fazes', table_data={ 'id1': array([1, 2, 3]), 'id2': array([1, 2, 1]) }) storage.write_table(table_name='fazdistrs', table_data={ 'my_variable': array([40, 50]), 'id2': array([1, 2]) }) ds = Dataset(in_storage=storage, in_table_name='gridcells', id_name="id0", dataset_name="mygridcell") ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone") ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz") ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr") dataset_pool = DatasetPool() dataset_pool._add_dataset('mygridcell', ds) dataset_pool._add_dataset('myzone', ds0) dataset_pool._add_dataset('myfaz', ds1) dataset_pool._add_dataset('myfazdistr', ds2) values = ds.compute_variables([ "mygridcell.disaggregate(myfazdistr.my_variable, intermediates=[myfaz,myzone])" ], dataset_pool=dataset_pool) should_be = array([40, 50, 40, 40, 50, 50, 40, 40, 40]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_two_levels")
def test_disaggregate(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='zones', table_data={ 'id':array([1,2,3,4]), 'id2':array([1,2,1,2]) } ) storage.write_table(table_name='faz', table_data={ 'my_variable':array([4,8]), 'id2':array([1,2]) } ) ds = Dataset(in_storage=storage, in_table_name='zones', id_name="id", dataset_name="myzone") ds2 = Dataset(in_storage=storage, in_table_name='faz', id_name="id2", dataset_name="myfaz") dataset_pool = DatasetPool() dataset_pool._add_dataset('myzone', ds) dataset_pool._add_dataset('myfaz', ds2) values = ds.compute_variables(["myzone.disaggregate(10.0*myfaz.my_variable)"], dataset_pool=dataset_pool) should_be = array([40, 80, 40, 80]) self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate")