def test_input2(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='buildings',
table_data={
'building_id' :array([ 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12]),
'zone_id' :array([11,11,21,21,31,31,11,11,21,21,31,31]),
'building_type_id' :array([ 7,99, 7,99, 7,99, 0,99, 0, 7,99, 7]),
'tenure' :array([ 1,1, 2,2, 1,2, 1,1, 2, 2,2, 1]),
}
)
ds = DatasetFactory().get_dataset('submarket',
package='bayarea',
arguments={'in_storage':storage}
)
self.assertEqual(ds.get_id_name(), ['submarket_id'])
self.assertTrue( "zone_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["zone_id"], array([11,11,11,21,21,21,31,31]))
self.assertTrue( "building_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["building_type_id"], array([0,7,99,0,7,99,7,99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(ds.get_id_attribute(), array([11001,
11071,
11991,
21002,
21072,
21992,
31071,
31992])
)
def test_input2(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='parcels',
table_data={
'parcel_id' :array([ 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12]),
'large_area_id' :array([11,11,21,21,31,31,11,11,21,21,31,31]),
'land_use_type_id' :array([ 7,99, 7,99, 7,99, 0,99, 0, 7,99, 7]),
}
)
ds = DatasetFactory().get_dataset('submarket',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(ds.get_id_name(), ['submarket_id'])
self.assertTrue( "large_area_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["large_area_id"], array([11,11,11,21,21,21,31,31]))
self.assertTrue( "land_use_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["land_use_type_id"], array([0,7,99,0,7,99,7,99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(ds.get_id_attribute(), array([
1100,
1107,
1199,
2100,
2107,
2199,
3107,
3199])
)
def test_input2(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='parcels',
table_data={
'parcel_id':
array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
'large_area_id':
array([11, 11, 21, 21, 31, 31, 11, 11, 21, 21, 31, 31]),
'land_use_type_id':
array([7, 99, 7, 99, 7, 99, 0, 99, 0, 7, 99, 7]),
})
ds = DatasetFactory().get_dataset('submarket',
package='psrc_parcel',
arguments={'in_storage': storage})
self.assertEqual(ds.get_id_name(), ['submarket_id'])
self.assertTrue("large_area_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["large_area_id"],
array([11, 11, 11, 21, 21, 21, 31, 31]))
self.assertTrue("land_use_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["land_use_type_id"],
array([0, 7, 99, 0, 7, 99, 7, 99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(
ds.get_id_attribute(),
array([1100, 1107, 1199, 2100, 2107, 2199, 3107, 3199]))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='districts',
table_data={
'district_id' :array([1,3,4]),
}
)
dc = DatasetFactory().get_dataset('district_commute',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['commute_id'])
self.assertTrue( "origin_district_id" in dc.get_known_attribute_names())
self.assertTrue( "destination_district_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([1001,
1003,
1004,
3001,
3003,
3004,
4001,
4003,
4004])
))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='households',
table_data={
'household_id' :array([1,2,3]),
'persons' :array([1,3,4]),
}
)
dc = DatasetFactory().get_dataset('faz_persons',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "persons" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='households',
table_data={
'household_id' :array([1,2,3]),
'persons' :array([1,3,4]),
}
)
dc = DatasetFactory().get_dataset('faz_persons',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "persons" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def test_input(self):
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="districts", table_data={"district_id": array([1, 3, 4])})
dc = DatasetFactory().get_dataset("district_commute", package="psrc_parcel", arguments={"in_storage": storage})
self.assertEqual(dc.get_id_name(), ["commute_id"])
self.assertTrue("origin_district_id" in dc.get_known_attribute_names())
self.assertTrue("destination_district_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004])))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='districts',
table_data={
'district_id': array([1, 3, 4]),
})
dc = DatasetFactory().get_dataset('district_commute',
package='psrc_parcel',
arguments={'in_storage': storage})
self.assertEqual(dc.get_id_name(), ['commute_id'])
self.assertTrue("origin_district_id" in dc.get_known_attribute_names())
self.assertTrue(
"destination_district_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(
allclose(
dc.get_id_attribute(),
array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004])))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_sectors',
table_data={
'sector_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_adhoc_sector_group_definitions',
table_data={
'sector_id' :array([]),
'group_id' :array([])
}
)
dc = DatasetFactory().get_dataset('faz_sector',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "sector_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_sectors',
table_data={
'sector_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_adhoc_sector_group_definitions',
table_data={
'sector_id' :array([]),
'group_id' :array([])
}
)
dc = DatasetFactory().get_dataset('faz_sector',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "sector_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def test_input2(self):
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="buildings",
table_data={
"building_id": array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
"zone_id": array([11, 11, 21, 21, 31, 31, 11, 11, 21, 21, 31, 31]),
"building_type_id": array([7, 99, 7, 99, 7, 99, 0, 99, 0, 7, 99, 7]),
"tenure": array([1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 1]),
},
)
ds = DatasetFactory().get_dataset("submarket", package="bayarea", arguments={"in_storage": storage})
self.assertEqual(ds.get_id_name(), ["submarket_id"])
self.assertTrue("zone_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["zone_id"], array([11, 11, 11, 21, 21, 21, 31, 31]))
self.assertTrue("building_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["building_type_id"], array([0, 7, 99, 0, 7, 99, 7, 99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(ds.get_id_attribute(), array([11001, 11071, 11991, 21002, 21072, 21992, 31071, 31992]))
def __init__(self, file_path, package_order, test_data):
"""
file_path is the directory path to the module defining the variable
to be tested. The name of the variable to be tested is extracted
from this path. As a special case, if the file_path ends in tests/variablename.py,
we assume that this is a tests subdirectory with just a unit test, and
remove the tests/ part to find the module name.
package_order is the sequence of Opus packages in which to look for the
dataset class modules for the datasets used in this test.
test_data is a dictionary of data for the datasets to use in this test,
where the key is the name of the dataset and the values are
dictionaries containing the attribute data in numpy containers,
e.g.:
test_data={
'gridcell':{
'grid_id': array([1, 2]),
'attribute_1': array([10, 20]),
},
'household':{
'household_id': array([1, 2]),
'grid_id': array([1, 2]),
},
}
If the dataset does not its own class to be created with, it must have
an attribute 'id' which is the unique identifier of the dataset.
"""
(dirname, filename) = os.path.split(file_path)
(front, lastdir) = os.path.split(dirname)
if lastdir == 'tests':
self.file_path = os.path.join(front, filename)
else:
self.file_path = file_path
storage = StorageFactory().get_storage('dict_storage')
self.dataset_pool = DatasetPool(package_order=package_order)
for dataset_name, attribute_dict in test_data.iteritems():
storage.write_table(table_name=dataset_name,
table_data=attribute_dict)
for dataset_name, attribute_dict in test_data.iteritems(
): # the two loops are needed because
dataset = None # of possible dependencies of one dataset on another
for package in package_order:
try: # try to create a specific dataset using given packages
dataset = DatasetFactory().get_dataset(dataset_name,
package=package,
arguments={
"in_table_name":
dataset_name,
"in_storage":
storage
})
break
except:
local_resources = {
'dataset_name': dataset_name,
'in_table_name': dataset_name,
'package': package,
'in_storage': storage
#'values': attribute_dict
}
try:
dataset = DatasetFactory().get_dataset(
dataset_name,
package=package,
arguments={"resources": local_resources})
break
except:
pass
if dataset is None:
try: # try to create general dataset
dataset = Dataset(dataset_name=dataset_name,
id_name=dataset_name + "_id",
in_table_name=dataset_name,
in_storage=storage)
if dataset_name + '_id' not in dataset.get_known_attribute_names(
):
dataset = Dataset(dataset_name=dataset_name,
id_name="id",
in_table_name=dataset_name,
in_storage=storage)
except:
raise StandardError, "Error in creating dataset %s." % dataset_name
self.dataset_pool._add_dataset(dataset_name, dataset)
def __init__(self, file_path, package_order, test_data):
"""
file_path is the directory path to the module defining the variable
to be tested. The name of the variable to be tested is extracted
from this path. As a special case, if the file_path ends in tests/variablename.py,
we assume that this is a tests subdirectory with just a unit test, and
remove the tests/ part to find the module name.
package_order is the sequence of Opus packages in which to look for the
dataset class modules for the datasets used in this test.
test_data is a dictionary of data for the datasets to use in this test,
where the key is the name of the dataset and the values are
dictionaries containing the attribute data in numpy containers,
e.g.:
test_data={
'gridcell':{
'grid_id': array([1, 2]),
'attribute_1': array([10, 20]),
},
'household':{
'household_id': array([1, 2]),
'grid_id': array([1, 2]),
},
}
If the dataset does not its own class to be created with, it must have
an attribute 'id' which is the unique identifier of the dataset.
"""
(dirname, filename) = os.path.split(file_path)
(front, lastdir) = os.path.split(dirname)
if lastdir=='tests':
self.file_path = os.path.join(front,filename)
else:
self.file_path = file_path
storage = StorageFactory().get_storage('dict_storage')
self.dataset_pool = DatasetPool(package_order=package_order)
for dataset_name, attribute_dict in test_data.iteritems():
storage.write_table(table_name=dataset_name, table_data=attribute_dict)
for dataset_name, attribute_dict in test_data.iteritems(): # the two loops are needed because
dataset = None # of possible dependencies of one dataset on another
for package in package_order:
try: # try to create a specific dataset using given packages
dataset = DatasetFactory().get_dataset(dataset_name, package=package,
arguments={"in_table_name": dataset_name,
"in_storage": storage})
break
except:
local_resources = {
'dataset_name': dataset_name,
'in_table_name': dataset_name,
'package': package,
'in_storage': storage
#'values': attribute_dict
}
try:
dataset = DatasetFactory().get_dataset(dataset_name, package=package,
arguments={"resources": local_resources})
break
except:
pass
if dataset is None:
try: # try to create general dataset
dataset = Dataset(dataset_name=dataset_name,
id_name=dataset_name+"_id",
in_table_name=dataset_name,
in_storage=storage)
if dataset_name+'_id' not in dataset.get_known_attribute_names():
dataset = Dataset(dataset_name=dataset_name,
id_name="id",
in_table_name=dataset_name,
in_storage=storage)
except:
raise StandardError, "Error in creating dataset %s." % dataset_name
self.dataset_pool._add_dataset(dataset_name, dataset)
