def test_input2(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='buildings', table_data={ 'building_id' :array([ 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12]), 'zone_id' :array([11,11,21,21,31,31,11,11,21,21,31,31]), 'building_type_id' :array([ 7,99, 7,99, 7,99, 0,99, 0, 7,99, 7]), 'tenure' :array([ 1,1, 2,2, 1,2, 1,1, 2, 2,2, 1]), } ) ds = DatasetFactory().get_dataset('submarket', package='bayarea', arguments={'in_storage':storage} ) self.assertEqual(ds.get_id_name(), ['submarket_id']) self.assertTrue( "zone_id" in ds.get_known_attribute_names()) self.assertArraysEqual( ds["zone_id"], array([11,11,11,21,21,21,31,31])) self.assertTrue( "building_type_id" in ds.get_known_attribute_names()) self.assertArraysEqual( ds["building_type_id"], array([0,7,99,0,7,99,7,99])) self.assertEqual(ds.size(), 8) self.assertArraysEqual(ds.get_id_attribute(), array([11001, 11071, 11991, 21002, 21072, 21992, 31071, 31992]) )
def test_input2(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='parcels', table_data={ 'parcel_id' :array([ 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12]), 'large_area_id' :array([11,11,21,21,31,31,11,11,21,21,31,31]), 'land_use_type_id' :array([ 7,99, 7,99, 7,99, 0,99, 0, 7,99, 7]), } ) ds = DatasetFactory().get_dataset('submarket', package='psrc_parcel', arguments={'in_storage':storage} ) self.assertEqual(ds.get_id_name(), ['submarket_id']) self.assertTrue( "large_area_id" in ds.get_known_attribute_names()) self.assertArraysEqual( ds["large_area_id"], array([11,11,11,21,21,21,31,31])) self.assertTrue( "land_use_type_id" in ds.get_known_attribute_names()) self.assertArraysEqual( ds["land_use_type_id"], array([0,7,99,0,7,99,7,99])) self.assertEqual(ds.size(), 8) self.assertArraysEqual(ds.get_id_attribute(), array([ 1100, 1107, 1199, 2100, 2107, 2199, 3107, 3199]) )
def test_input2(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='parcels', table_data={ 'parcel_id': array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), 'large_area_id': array([11, 11, 21, 21, 31, 31, 11, 11, 21, 21, 31, 31]), 'land_use_type_id': array([7, 99, 7, 99, 7, 99, 0, 99, 0, 7, 99, 7]), }) ds = DatasetFactory().get_dataset('submarket', package='psrc_parcel', arguments={'in_storage': storage}) self.assertEqual(ds.get_id_name(), ['submarket_id']) self.assertTrue("large_area_id" in ds.get_known_attribute_names()) self.assertArraysEqual(ds["large_area_id"], array([11, 11, 11, 21, 21, 21, 31, 31])) self.assertTrue("land_use_type_id" in ds.get_known_attribute_names()) self.assertArraysEqual(ds["land_use_type_id"], array([0, 7, 99, 0, 7, 99, 7, 99])) self.assertEqual(ds.size(), 8) self.assertArraysEqual( ds.get_id_attribute(), array([1100, 1107, 1199, 2100, 2107, 2199, 3107, 3199]))
def test_input(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='districts', table_data={ 'district_id' :array([1,3,4]), } ) dc = DatasetFactory().get_dataset('district_commute', package='psrc_parcel', arguments={'in_storage':storage} ) self.assertEqual(dc.get_id_name(), ['commute_id']) self.assertTrue( "origin_district_id" in dc.get_known_attribute_names()) self.assertTrue( "destination_district_id" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue(allclose(dc.get_id_attribute(), array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004]) ))
def test_input(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='fazes', table_data={ 'faz_id' :array([1,3,4]), } ) storage.write_table(table_name='households', table_data={ 'household_id' :array([1,2,3]), 'persons' :array([1,3,4]), } ) dc = DatasetFactory().get_dataset('faz_persons', package='psrc_parcel', arguments={'in_storage':storage} ) self.assertEqual(dc.get_id_name(), ['dummy_id']) self.assertTrue( "faz_id" in dc.get_known_attribute_names()) self.assertTrue( "persons" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue(allclose(dc.get_id_attribute(), array([ 101, 103, 104, 301, 303, 304, 401, 403, 404]) ))
def test_input(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='fazes', table_data={ 'faz_id' :array([1,3,4]), } ) storage.write_table(table_name='households', table_data={ 'household_id' :array([1,2,3]), 'persons' :array([1,3,4]), } ) dc = DatasetFactory().get_dataset('faz_persons', package='psrc_parcel', arguments={'in_storage':storage} ) self.assertEqual(dc.get_id_name(), ['dummy_id']) self.assertTrue( "faz_id" in dc.get_known_attribute_names()) self.assertTrue( "persons" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue(allclose(dc.get_id_attribute(), array([ 101, 103, 104, 301, 303, 304, 401, 403, 404]) ))
def test_input(self): storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="districts", table_data={"district_id": array([1, 3, 4])}) dc = DatasetFactory().get_dataset("district_commute", package="psrc_parcel", arguments={"in_storage": storage}) self.assertEqual(dc.get_id_name(), ["commute_id"]) self.assertTrue("origin_district_id" in dc.get_known_attribute_names()) self.assertTrue("destination_district_id" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue(allclose(dc.get_id_attribute(), array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004])))
def test_input(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='districts', table_data={ 'district_id': array([1, 3, 4]), }) dc = DatasetFactory().get_dataset('district_commute', package='psrc_parcel', arguments={'in_storage': storage}) self.assertEqual(dc.get_id_name(), ['commute_id']) self.assertTrue("origin_district_id" in dc.get_known_attribute_names()) self.assertTrue( "destination_district_id" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue( allclose( dc.get_id_attribute(), array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004])))
def test_input(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='fazes', table_data={ 'faz_id' :array([1,3,4]), } ) storage.write_table(table_name='employment_sectors', table_data={ 'sector_id' :array([1,3,4]), } ) storage.write_table(table_name='employment_adhoc_sector_group_definitions', table_data={ 'sector_id' :array([]), 'group_id' :array([]) } ) dc = DatasetFactory().get_dataset('faz_sector', package='psrc_parcel', arguments={'in_storage':storage} ) self.assertEqual(dc.get_id_name(), ['dummy_id']) self.assertTrue( "faz_id" in dc.get_known_attribute_names()) self.assertTrue( "sector_id" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue(allclose(dc.get_id_attribute(), array([ 101, 103, 104, 301, 303, 304, 401, 403, 404]) ))
def test_input(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='fazes', table_data={ 'faz_id' :array([1,3,4]), } ) storage.write_table(table_name='employment_sectors', table_data={ 'sector_id' :array([1,3,4]), } ) storage.write_table(table_name='employment_adhoc_sector_group_definitions', table_data={ 'sector_id' :array([]), 'group_id' :array([]) } ) dc = DatasetFactory().get_dataset('faz_sector', package='psrc_parcel', arguments={'in_storage':storage} ) self.assertEqual(dc.get_id_name(), ['dummy_id']) self.assertTrue( "faz_id" in dc.get_known_attribute_names()) self.assertTrue( "sector_id" in dc.get_known_attribute_names()) self.assertEqual(dc.size(), 9) self.assertTrue(allclose(dc.get_id_attribute(), array([ 101, 103, 104, 301, 303, 304, 401, 403, 404]) ))
def test_input2(self): storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="buildings", table_data={ "building_id": array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), "zone_id": array([11, 11, 21, 21, 31, 31, 11, 11, 21, 21, 31, 31]), "building_type_id": array([7, 99, 7, 99, 7, 99, 0, 99, 0, 7, 99, 7]), "tenure": array([1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 1]), }, ) ds = DatasetFactory().get_dataset("submarket", package="bayarea", arguments={"in_storage": storage}) self.assertEqual(ds.get_id_name(), ["submarket_id"]) self.assertTrue("zone_id" in ds.get_known_attribute_names()) self.assertArraysEqual(ds["zone_id"], array([11, 11, 11, 21, 21, 21, 31, 31])) self.assertTrue("building_type_id" in ds.get_known_attribute_names()) self.assertArraysEqual(ds["building_type_id"], array([0, 7, 99, 0, 7, 99, 7, 99])) self.assertEqual(ds.size(), 8) self.assertArraysEqual(ds.get_id_attribute(), array([11001, 11071, 11991, 21002, 21072, 21992, 31071, 31992]))
def __init__(self, file_path, package_order, test_data): """ file_path is the directory path to the module defining the variable to be tested. The name of the variable to be tested is extracted from this path. As a special case, if the file_path ends in tests/variablename.py, we assume that this is a tests subdirectory with just a unit test, and remove the tests/ part to find the module name. package_order is the sequence of Opus packages in which to look for the dataset class modules for the datasets used in this test. test_data is a dictionary of data for the datasets to use in this test, where the key is the name of the dataset and the values are dictionaries containing the attribute data in numpy containers, e.g.: test_data={ 'gridcell':{ 'grid_id': array([1, 2]), 'attribute_1': array([10, 20]), }, 'household':{ 'household_id': array([1, 2]), 'grid_id': array([1, 2]), }, } If the dataset does not its own class to be created with, it must have an attribute 'id' which is the unique identifier of the dataset. """ (dirname, filename) = os.path.split(file_path) (front, lastdir) = os.path.split(dirname) if lastdir == 'tests': self.file_path = os.path.join(front, filename) else: self.file_path = file_path storage = StorageFactory().get_storage('dict_storage') self.dataset_pool = DatasetPool(package_order=package_order) for dataset_name, attribute_dict in test_data.iteritems(): storage.write_table(table_name=dataset_name, table_data=attribute_dict) for dataset_name, attribute_dict in test_data.iteritems( ): # the two loops are needed because dataset = None # of possible dependencies of one dataset on another for package in package_order: try: # try to create a specific dataset using given packages dataset = DatasetFactory().get_dataset(dataset_name, package=package, arguments={ "in_table_name": dataset_name, "in_storage": storage }) break except: local_resources = { 'dataset_name': dataset_name, 'in_table_name': dataset_name, 'package': package, 'in_storage': storage #'values': attribute_dict } try: dataset = DatasetFactory().get_dataset( dataset_name, package=package, arguments={"resources": local_resources}) break except: pass if dataset is None: try: # try to create general dataset dataset = Dataset(dataset_name=dataset_name, id_name=dataset_name + "_id", in_table_name=dataset_name, in_storage=storage) if dataset_name + '_id' not in dataset.get_known_attribute_names( ): dataset = Dataset(dataset_name=dataset_name, id_name="id", in_table_name=dataset_name, in_storage=storage) except: raise StandardError, "Error in creating dataset %s." % dataset_name self.dataset_pool._add_dataset(dataset_name, dataset)
def __init__(self, file_path, package_order, test_data): """ file_path is the directory path to the module defining the variable to be tested. The name of the variable to be tested is extracted from this path. As a special case, if the file_path ends in tests/variablename.py, we assume that this is a tests subdirectory with just a unit test, and remove the tests/ part to find the module name. package_order is the sequence of Opus packages in which to look for the dataset class modules for the datasets used in this test. test_data is a dictionary of data for the datasets to use in this test, where the key is the name of the dataset and the values are dictionaries containing the attribute data in numpy containers, e.g.: test_data={ 'gridcell':{ 'grid_id': array([1, 2]), 'attribute_1': array([10, 20]), }, 'household':{ 'household_id': array([1, 2]), 'grid_id': array([1, 2]), }, } If the dataset does not its own class to be created with, it must have an attribute 'id' which is the unique identifier of the dataset. """ (dirname, filename) = os.path.split(file_path) (front, lastdir) = os.path.split(dirname) if lastdir=='tests': self.file_path = os.path.join(front,filename) else: self.file_path = file_path storage = StorageFactory().get_storage('dict_storage') self.dataset_pool = DatasetPool(package_order=package_order) for dataset_name, attribute_dict in test_data.iteritems(): storage.write_table(table_name=dataset_name, table_data=attribute_dict) for dataset_name, attribute_dict in test_data.iteritems(): # the two loops are needed because dataset = None # of possible dependencies of one dataset on another for package in package_order: try: # try to create a specific dataset using given packages dataset = DatasetFactory().get_dataset(dataset_name, package=package, arguments={"in_table_name": dataset_name, "in_storage": storage}) break except: local_resources = { 'dataset_name': dataset_name, 'in_table_name': dataset_name, 'package': package, 'in_storage': storage #'values': attribute_dict } try: dataset = DatasetFactory().get_dataset(dataset_name, package=package, arguments={"resources": local_resources}) break except: pass if dataset is None: try: # try to create general dataset dataset = Dataset(dataset_name=dataset_name, id_name=dataset_name+"_id", in_table_name=dataset_name, in_storage=storage) if dataset_name+'_id' not in dataset.get_known_attribute_names(): dataset = Dataset(dataset_name=dataset_name, id_name="id", in_table_name=dataset_name, in_storage=storage) except: raise StandardError, "Error in creating dataset %s." % dataset_name self.dataset_pool._add_dataset(dataset_name, dataset)