def _try_load_district_dataset(self, **kwargs):
try:
districts = DatasetFactory().get_dataset("district", package="psrc_parcel", arguments=kwargs)
districts.load_dataset()
except:
districts = None
return districts
def prepare_for_run(self,
scheduled_events_dataset_name=None,
scheduled_events_table=None,
scheduled_events_storage=None):
if (scheduled_events_storage is None) or (
(scheduled_events_table is None) and
(scheduled_events_dataset_name is None)):
## this should not happen
dataset_pool = SessionConfiguration().get_dataset_pool()
self.scheduled_events = dataset_pool.get_dataset(
'scheduled_%s_events' % self.dataset.get_dataset_name())
return self.scheduled_events
if not scheduled_events_dataset_name:
scheduled_events_dataset_name = DatasetFactory(
).dataset_name_for_table(scheduled_events_table)
self.scheduled_events = DatasetFactory().search_for_dataset(
scheduled_events_dataset_name,
package_order=SessionConfiguration().package_order,
arguments={
'in_storage': scheduled_events_storage,
'in_table_name': scheduled_events_table,
'id_name': []
})
return self.scheduled_events
def _load_new_dataset(self, dataset_name, dataset_arguments):
"""Create a new dataset object and put it in the pool.
Use first class found in the 'datasets' directory
for the packages in package_order."""
arguments = deepcopy(dataset_arguments)
# Augment arguments as needed.
if '_x_' in dataset_name:
dataset_names = dataset_name.split('_x_')
for i in range(len(dataset_names)):
key = 'dataset%s' % (i + 1)
arguments[key] = self.get_dataset(dataset_names[i])
else:
arguments['in_storage'] = self._storage
if len(dataset_arguments.get('id_name', ['dummy'])) == 0:
dataset = DatasetFactory().search_for_dataset_with_hidden_id(
dataset_name,
self._package_order,
subdir='datasets',
arguments=arguments)
else:
dataset = DatasetFactory().search_for_dataset(dataset_name,
self._package_order,
subdir='datasets',
arguments=arguments)
self._add_dataset(dataset_name, dataset)
def __init__(self, variable_name, observed_data, filename=None, transformation=None, inverse_transformation=None,
filter=None, match=False, dependent_datasets={}, **kwargs):
""" 'variable_name' is a quantity about which we have data available.
'observed_data' is of type ObservedData, it is the grouping parent.
'filename' is the name of file where
the data is stored. It can be None, if the observed_data.directory is a cache.
'transformation' is an operation to be performed on the data (e.g. sqrt, log),
'inverse_transformation' is the inverse function of 'transformation'. If it not given, it
is determined automatically.
'filter' is a variable that will be applied to both, the observed data and the simulated data.
'match' (logical) determines if the dataset should be matched (by ids) with the simulated dataset. Elements
that don't match are eliminated from the simulated dataset.
'dependent_datasets' (if any) should be a dictionary of dataset_name:{'filename': filename, 'match': True|False, **kwargs}.
They will be added to the dataset_pool.
Remaining arguments are passed into DatasetFactory, thus it can contain information about how
to create the corresponding dataset.
"""
self.variable_name = VariableName(variable_name)
self.dataset_name = self.variable_name.get_dataset_name()
dataset_pool = observed_data.get_dataset_pool()
self.matching_datasets = {}
if dataset_pool is None:
kwargs.update({'in_storage':observed_data.get_storage(), 'in_table_name': filename})
try:
self.dataset = DatasetFactory().search_for_dataset(self.dataset_name, observed_data.get_package_order(), arguments=kwargs)
except: # take generic dataset
self.dataset = Dataset(dataset_name=self.dataset_name, **kwargs)
else:
self.dataset = dataset_pool.get_dataset(self.dataset_name)
if match:
self.add_match(self.dataset)
for dep_dataset_name, info in dependent_datasets.iteritems():
if dataset_pool is None:
dataset_pool = DatasetPool(storage=observed_data.get_storage(), package_order=observed_data.get_package_order())
info.update({'in_storage':observed_data.get_storage(), 'in_table_name': info.get('filename')})
del info['filename']
match = False
if 'match' in info.keys():
match = info['match']
del info['match']
try:
dep_dataset = DatasetFactory().search_for_dataset(dep_dataset_name, observed_data.get_package_order(), arguments=info)
except:
dep_dataset = Dataset(dataset_name=dep_dataset_name, **info)
dataset_pool.replace_dataset(dep_dataset_name, dep_dataset)
if match:
self.add_match(dep_dataset)
if self.variable_name.get_alias() not in self.dataset.get_known_attribute_names():
self.dataset.compute_variables([self.variable_name], dataset_pool=dataset_pool)
if filter is not None:
filter_values = self.dataset.compute_variables([filter], dataset_pool=dataset_pool)
idx = where(filter_values > 0)[0]
self.add_match(self.dataset, idx)
self.dataset.subset_by_index(idx)
self.transformation = transformation
self.inverse_transformation = inverse_transformation
if (self.transformation is not None) and (self.inverse_transformation is None):
self.inverse_transformation = self.transformation_pairs[self.transformation]
def _try_load_district_dataset(self, **kwargs):
try:
districts = DatasetFactory().get_dataset("district",
package='psrc_parcel',
arguments=kwargs)
districts.load_dataset()
except:
districts = None
return districts
def create_view(database, table_to_link_name, dataset_name):
df = DatasetFactory()
spatial_table_name = '%s_shp' % df._table_module_class_names_for_dataset(
dataset_name)[0]
try:
spatial_table = database.get_table(spatial_table_name)
except:
print 'Error, could not create view because spatial table %s could not be found' % spatial_table_name
return
table_to_link = database.get_table(table_to_link_name)
spatial_primary_keys = database.get_primary_keys_for_table(spatial_table)
table_to_link_primary_keys = database.get_primary_keys_for_table(
table_to_link)
primary_key_error_msg = 'The table %s either has no primary key or has a composite primary key. View creation does not support such tables'
if spatial_primary_keys == [] or len(spatial_primary_keys) > 1 is not None:
raise Exception(primary_key_error_msg % (spatial_table_name))
if table_to_link_primary_keys == [] or len(
table_to_link_primary_keys) > 1 is not None:
raise Exception(primary_key_error_msg % (table_to_link_name))
cols_from_spatial = [c.name for c in spatial_table.c]
cols_from_linked = [
c.name for c in table_to_link.c if c.name not in cols_from_spatial
]
if table_to_link_primary_keys[0].name not in cols_from_spatial:
print 'WARNING: view will not be able to be created because the spatial table does not contain the column %s' % table_to_link_primary_keys[
0].name
cols = ','.join(['s.%s' % c for c in cols_from_spatial] +
['l.%s' % c for c in cols_from_linked])
params = {
'join_col': table_to_link_primary_keys[0].name,
'to_link': table_to_link,
'spatial_table': spatial_table_name,
'cols': cols,
'spatial_key': table_to_link_primary_keys[0].name
}
qry = ('''
CREATE OR REPLACE VIEW %(to_link)s_view
AS SELECT %(cols)s from %(to_link)s as l, %(spatial_table)s as s
WHERE l.%(join_col)s=s.%(spatial_key)s
''' % params)
try:
database.execute(qry)
print qry
except:
print 'Error, could not create view'
print qry
import traceback
traceback.print_exc()
def solve_dependencies(self, **kwargs):
variable_names = [VariableName(v) for v in self.submarket_definition]
ds = [vn.get_dataset_name() for vn in variable_names]
short_names = [vn.get_alias() for vn in variable_names]
assert len(set(ds))==1 ## the submarket_definition should be of the same dataset
dataset = DatasetFactory().get_dataset(ds[0],
package='urbansim_parcel',
arguments=kwargs)
dataset.compute_variables(self.submarket_definition)
submarket_ids = generate_unique_ids(dataset, short_names)
return (dataset, short_names, submarket_ids)
def solve_dependencies(self, **kwargs):
variable_names = [VariableName(v) for v in self.subgroup_definition]
ds = [vn.get_dataset_name() for vn in variable_names]
short_names = [vn.get_alias() for vn in variable_names]
assert len(set(ds))==1 ## the subgroup_definition should be of the same dataset
dataset = DatasetFactory().search_for_dataset(ds[0],
package_order=self.default_package_order,
arguments=kwargs)
dataset.compute_variables(self.subgroup_definition)
subgroup_ids, multipler = generate_unique_ids(dataset, short_names)
self.multipler = multipler
return (dataset, short_names, subgroup_ids)
def _get_data(self, year, dataset_name, attribute_name):
current_year = SimulationState().get_current_time()
SimulationState().set_current_time(year)
dataset = DatasetFactory().get_dataset(dataset_name, package='urbansim',
subdir='datasets',
arguments={'in_storage':AttributeCache()})
dataset.compute_variables(attribute_name,
resources=self.simulation.config)
variable_name = VariableName(attribute_name)
short_name = variable_name.get_short_name()
values = dataset.get_attribute(short_name)
SimulationState().set_current_time(current_year)
return values
def tmp_skip_test_gridcell_unrolling_changes_development_type_id(self):
"""Does unrolling update development_type_id?
"""
# Force one grid cell to be "vacant", so can check that development_type_id changes.
cache_directory = SimulationState().get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
development_event_history = DatasetFactory().get_dataset(
'development_event_history',
package='urbansim',
subdir='datasets',
arguments={
'in_storage':
StorageFactory().get_storage('flt_storage',
storage_location=flt_directory)
})
changed_grid_id = 10123
new_row = {
'grid_id': array([changed_grid_id]),
'scheduled_year': array([self.base_year - 1]),
'residential_units': array([1000]),
'commercial_sqft': array([10000000]),
'industrial_sfft': array([10000000]),
'governmental_sqft': array([10000000]),
'starting_development_type_id': array([1000]),
}
development_event_history.add_elements(new_row,
require_all_attributes=False)
development_event_history.flush_dataset()
gridcells = SessionConfiguration().get_dataset_from_pool('gridcell')
development_event_history = SessionConfiguration(
).get_dataset_from_pool('development_event_history')
unroller = UnrollGridcells()
unroller.unroll_gridcells_to_cache(gridcells,
development_event_history,
cache_directory, self.base_year)
cache_directory = SimulationState().get_cache_directory()
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
gridcell = {}
for year in [1978, 1979]:
flt_directory = os.path.join(cache_directory, str(year))
gridcell[year] = DatasetFactory().get_dataset(
'gridcell',
package='urbansim',
subdir='datasets',
arguments={
'in_storage':
StorageFactory().get_storage(
'flt_storage', storage_location=flt_directory)
})
self.assertEqual(
gridcell[1978].get_attribute_by_id('development_type_id',
changed_grid_id), 1000)
self.assertNotEqual(
gridcell[1979].get_attribute_by_id('development_type_id',
changed_grid_id),
gridcell[1978].get_attribute_by_id('development_type_id',
changed_grid_id))
def solve_dependencies(self, **kwargs):
variable_names = [VariableName(v) for v in self.submarket_definition]
ds = [vn.get_dataset_name() for vn in variable_names]
short_names = [vn.get_alias() for vn in variable_names]
assert len(
set(ds)
) == 1 ## the submarket_definition should be of the same dataset
dataset = DatasetFactory().get_dataset(ds[0],
package='urbansim_parcel',
arguments=kwargs)
dataset.compute_variables(self.submarket_definition)
submarket_ids = generate_unique_ids(dataset, short_names)
return (dataset, short_names, submarket_ids)
def prepare_for_run(self, refinement_dataset_name=None,
refinement_storage=None,
refinement_table_name=None):
from opus_core.datasets.dataset_factory import DatasetFactory
from opus_core.session_configuration import SessionConfiguration
df = DatasetFactory()
if not refinement_dataset_name:
refinement_dataset_name = df.dataset_name_for_table(refinement_table_name)
refinement = df.search_for_dataset(refinement_dataset_name,
package_order=SessionConfiguration().package_order,
arguments={'in_storage':refinement_storage,
'in_table_name':refinement_table_name}
)
return refinement
def _get_data(self, year, dataset_name, attribute_name):
current_year = SimulationState().get_current_time()
SimulationState().set_current_time(year)
dataset = DatasetFactory().get_dataset(
dataset_name,
package='urbansim',
subdir='datasets',
arguments={'in_storage': AttributeCache()})
dataset.compute_variables(attribute_name,
resources=self.simulation.config)
variable_name = VariableName(attribute_name)
short_name = variable_name.get_short_name()
values = dataset.get_attribute(short_name)
SimulationState().set_current_time(current_year)
return values
def setUp(self):
self.config = TestCacheConfiguration()
self.simulation_state = SimulationState(new_instance=True)
SessionConfiguration(self.config, new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = self.config['base_year']
creating_baseyear_cache_configuration = self.config['creating_baseyear_cache_configuration']
self.simulation_state.set_current_time(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
copytree(os.path.join(creating_baseyear_cache_configuration.baseyear_cache.existing_cache_to_copy,
str(self.base_year)),
os.path.join(cache_directory, str(self.base_year)))
cacher = CacheScenarioDatabase()
cacher.prepare_data_before_baseyear(cache_directory, self.base_year, creating_baseyear_cache_configuration)
self.config['cache_directory'] = cache_directory
cache_storage = AttributeCache().get_flt_storage_for_year(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
self.gridcell = DatasetFactory().get_dataset('gridcell',
package='urbansim',
subdir='datasets',
arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)}
)
def prepare_for_run(self,
what=None,
rate_dataset_name=None,
rate_storage=None,
rate_table=None,
sample_rates=False,
n=100,
multiplicator=1,
flush_rates=True):
"""
what - unused, argument kept to be compatible with old code
"""
from opus_core.datasets.dataset_factory import DatasetFactory
from opus_core.session_configuration import SessionConfiguration
if (rate_storage is None) or ((rate_table is None) and
(rate_dataset_name is None)):
return self.resources
if not rate_dataset_name:
rate_dataset_name = DatasetFactory().dataset_name_for_table(
rate_table)
rates = DatasetFactory().search_for_dataset(
rate_dataset_name,
package_order=SessionConfiguration().package_order,
arguments={
'in_storage': rate_storage,
'in_table_name': rate_table,
})
if sample_rates:
cache_storage = None
if flush_rates:
cache_storage = rate_storage
rates.sample_rates(n=n,
cache_storage=cache_storage,
multiplicator=multiplicator)
self.resources.merge(
{rate_dataset_name: rates}
) #to be compatible with old-style one-relocation_probabilities-module-per-model
self.resources.merge({'relocation_rate': rates})
return self.resources
### In order to remove a circular dependency between this file and
### household_location_choice_model_creator, these unit tests were moved into
### urbansim.tests.test_agent_relocation_model.
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='households',
table_data={
'household_id' :array([1,2,3]),
'persons' :array([1,3,4]),
}
)
dc = DatasetFactory().get_dataset('faz_persons',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "persons" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def test_input2(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='parcels',
table_data={
'parcel_id':
array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
'large_area_id':
array([11, 11, 21, 21, 31, 31, 11, 11, 21, 21, 31, 31]),
'land_use_type_id':
array([7, 99, 7, 99, 7, 99, 0, 99, 0, 7, 99, 7]),
})
ds = DatasetFactory().get_dataset('submarket',
package='psrc_parcel',
arguments={'in_storage': storage})
self.assertEqual(ds.get_id_name(), ['submarket_id'])
self.assertTrue("large_area_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["large_area_id"],
array([11, 11, 11, 21, 21, 21, 31, 31]))
self.assertTrue("land_use_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["land_use_type_id"],
array([0, 7, 99, 0, 7, 99, 7, 99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(
ds.get_id_attribute(),
array([1100, 1107, 1199, 2100, 2107, 2199, 3107, 3199]))
def prepare_for_run(self,
refinement_dataset_name=None,
refinement_storage=None,
refinement_table_name=None):
from opus_core.datasets.dataset_factory import DatasetFactory
from opus_core.session_configuration import SessionConfiguration
df = DatasetFactory()
if not refinement_dataset_name:
refinement_dataset_name = df.dataset_name_for_table(
refinement_table_name)
refinement = df.search_for_dataset(
refinement_dataset_name,
package_order=SessionConfiguration().package_order,
arguments={
'in_storage': refinement_storage,
'in_table_name': refinement_table_name
})
return refinement
def prepare_for_run(self, dataset_name=None, table_name=None, storage=None):
if (storage is None) or ((table_name is None) and (dataset_name is None)):
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset = dataset_pool.get_dataset( 'target_vacancy' )
return dataset
if not dataset_name:
dataset_name = DatasetFactory().dataset_name_for_table(table_name)
dataset = DatasetFactory().search_for_dataset(dataset_name,
package_order=SessionConfiguration().package_order,
arguments={'in_storage':storage,
'in_table_name':table_name,
'id_name':[]
}
)
if self.target_vancy_dataset is None:
self.target_vancy_dataset = dataset
return dataset
def setUp(self):
self.config = TestCacheConfiguration()
self.simulation_state = SimulationState(new_instance=True)
SessionConfiguration(self.config,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = self.config['base_year']
creating_baseyear_cache_configuration = self.config[
'creating_baseyear_cache_configuration']
self.simulation_state.set_current_time(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
copytree(
os.path.join(
creating_baseyear_cache_configuration.baseyear_cache.
existing_cache_to_copy, str(self.base_year)),
os.path.join(cache_directory, str(self.base_year)))
cacher = CacheScenarioDatabase()
cacher.prepare_data_before_baseyear(
cache_directory, self.base_year,
creating_baseyear_cache_configuration)
self.config['cache_directory'] = cache_directory
cache_storage = AttributeCache().get_flt_storage_for_year(
self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
self.gridcell = DatasetFactory().get_dataset(
'gridcell',
package='urbansim',
subdir='datasets',
arguments={
'in_storage':
StorageFactory().get_storage('flt_storage',
storage_location=flt_directory)
})
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='districts',
table_data={
'district_id' :array([1,3,4]),
}
)
dc = DatasetFactory().get_dataset('district_commute',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['commute_id'])
self.assertTrue( "origin_district_id" in dc.get_known_attribute_names())
self.assertTrue( "destination_district_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([1001,
1003,
1004,
3001,
3003,
3004,
4001,
4003,
4004])
))
def test_input2(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='parcels',
table_data={
'parcel_id' :array([ 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12]),
'large_area_id' :array([11,11,21,21,31,31,11,11,21,21,31,31]),
'land_use_type_id' :array([ 7,99, 7,99, 7,99, 0,99, 0, 7,99, 7]),
}
)
ds = DatasetFactory().get_dataset('submarket',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(ds.get_id_name(), ['submarket_id'])
self.assertTrue( "large_area_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["large_area_id"], array([11,11,11,21,21,21,31,31]))
self.assertTrue( "land_use_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["land_use_type_id"], array([0,7,99,0,7,99,7,99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(ds.get_id_attribute(), array([
1100,
1107,
1199,
2100,
2107,
2199,
3107,
3199])
)
def test_input2(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='buildings',
table_data={
'building_id' :array([ 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12]),
'zone_id' :array([11,11,21,21,31,31,11,11,21,21,31,31]),
'building_type_id' :array([ 7,99, 7,99, 7,99, 0,99, 0, 7,99, 7]),
'tenure' :array([ 1,1, 2,2, 1,2, 1,1, 2, 2,2, 1]),
}
)
ds = DatasetFactory().get_dataset('submarket',
package='bayarea',
arguments={'in_storage':storage}
)
self.assertEqual(ds.get_id_name(), ['submarket_id'])
self.assertTrue( "zone_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["zone_id"], array([11,11,11,21,21,21,31,31]))
self.assertTrue( "building_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual( ds["building_type_id"], array([0,7,99,0,7,99,7,99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(ds.get_id_attribute(), array([11001,
11071,
11991,
21002,
21072,
21992,
31071,
31992])
)
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='households',
table_data={
'household_id' :array([1,2,3]),
'persons' :array([1,3,4]),
}
)
dc = DatasetFactory().get_dataset('faz_persons',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "persons" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def prepare_for_run(self, what=None,
rate_dataset_name=None,
rate_storage=None,
rate_table=None,
sample_rates=False,
n=100,
multiplicator=1,
flush_rates=True):
"""
what - unused, argument kept to be compatible with old code
"""
from opus_core.datasets.dataset_factory import DatasetFactory
from opus_core.session_configuration import SessionConfiguration
if (rate_storage is None) or ((rate_table is None) and (rate_dataset_name is None)):
return self.resources
if not rate_dataset_name:
rate_dataset_name = DatasetFactory().dataset_name_for_table(rate_table)
rates = DatasetFactory().search_for_dataset(rate_dataset_name,
package_order=SessionConfiguration().package_order,
arguments={'in_storage':rate_storage,
'in_table_name':rate_table,
}
)
if sample_rates:
cache_storage=None
if flush_rates:
cache_storage=rate_storage
rates.sample_rates(n=n, cache_storage=cache_storage,
multiplicator=multiplicator)
self.resources.merge({rate_dataset_name:rates}) #to be compatible with old-style one-relocation_probabilities-module-per-model
self.resources.merge({'relocation_rate':rates})
return self.resources
### In order to remove a circular dependency between this file and
### household_location_choice_model_creator, these unit tests were moved into
### urbansim.tests.test_agent_relocation_model.
def _try_load_datasets(self, **kwargs):
try:
fazes = DatasetFactory().get_dataset("faz",
package='urbansim',
arguments=kwargs)
sectors = DatasetFactory().get_dataset("employment_sector",
package='urbansim',
arguments=kwargs)
fazes.load_dataset()
sectors.load_dataset(attributes='*')
except:
fazes = None
sectors = None
return (fazes, sectors)
def _try_load_datasets(self, **kwargs):
try:
fazes = DatasetFactory().get_dataset("faz",
package='urbansim',
arguments=kwargs)
households = DatasetFactory().get_dataset("household",
package='urbansim',
arguments=kwargs)
fazes.load_dataset()
households.load_dataset(attributes='*')
except:
fazes = None
households = None
return (fazes, households)
def prepare_for_run(self, what=None,
rate_dataset_name="rate",
rate_storage=None,
rate_table=None,
probability_attribute=None,
sample_rates=False,
n=100,
multiplicator=1,
flush_rates=True):
"""
what - unused, argument kept to be compatible with old code
"""
from opus_core.datasets.dataset_factory import DatasetFactory
from opus_core.session_configuration import SessionConfiguration
if (rate_storage is None) or ((rate_table is None) and (rate_dataset_name is None)):
return self.resources
if not rate_dataset_name:
rate_dataset_name = DatasetFactory().dataset_name_for_table(rate_table)
rates = DatasetFactory().search_for_dataset(rate_dataset_name,
package_order=SessionConfiguration().package_order,
arguments={'in_storage':rate_storage,
'in_table_name':rate_table,
}
)
if probability_attribute is not None:
rates.probability_attribute = probability_attribute
if sample_rates:
cache_storage=None
if flush_rates:
cache_storage=rate_storage
rates.sample_rates(n=n, cache_storage=cache_storage,
multiplicator=multiplicator)
self.resources.merge({rate_dataset_name:rates}) #to be compatible with old-style one-relocation_probabilities-module-per-model
self.resources.merge({'rate_set':rates})
return self.resources
def test_input(self):
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="districts", table_data={"district_id": array([1, 3, 4])})
dc = DatasetFactory().get_dataset("district_commute", package="psrc_parcel", arguments={"in_storage": storage})
self.assertEqual(dc.get_id_name(), ["commute_id"])
self.assertTrue("origin_district_id" in dc.get_known_attribute_names())
self.assertTrue("destination_district_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004])))
def _try_load_datasets(self, **kwargs):
try:
fazes = DatasetFactory().get_dataset("faz",
package='urbansim',
arguments=kwargs)
households = DatasetFactory().get_dataset("household",
package='urbansim',
arguments=kwargs)
fazes.load_dataset()
households.load_dataset(attributes='*')
except:
fazes = None
households = None
return (fazes, households)
def _try_load_datasets(self, **kwargs):
try:
fazes = DatasetFactory().get_dataset("faz",
package='urbansim',
arguments=kwargs)
sectors = DatasetFactory().get_dataset("employment_sector",
package='urbansim',
arguments=kwargs)
fazes.load_dataset()
sectors.load_dataset(attributes='*')
except:
fazes = None
sectors = None
return (fazes, sectors)
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='districts',
table_data={
'district_id': array([1, 3, 4]),
})
dc = DatasetFactory().get_dataset('district_commute',
package='psrc_parcel',
arguments={'in_storage': storage})
self.assertEqual(dc.get_id_name(), ['commute_id'])
self.assertTrue("origin_district_id" in dc.get_known_attribute_names())
self.assertTrue(
"destination_district_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(
allclose(
dc.get_id_attribute(),
array([1001, 1003, 1004, 3001, 3003, 3004, 4001, 4003, 4004])))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_sectors',
table_data={
'sector_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_adhoc_sector_group_definitions',
table_data={
'sector_id' :array([]),
'group_id' :array([])
}
)
dc = DatasetFactory().get_dataset('faz_sector',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "sector_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def tmp_skip_test_gridcell_unrolling_changes_development_type_id(self):
"""Does unrolling update development_type_id?
"""
# Force one grid cell to be "vacant", so can check that development_type_id changes.
cache_directory = SimulationState().get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
development_event_history = DatasetFactory().get_dataset('development_event_history',
package='urbansim',
subdir='datasets',
arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)}
)
changed_grid_id = 10123
new_row = {
'grid_id':array([changed_grid_id]),
'scheduled_year':array([self.base_year - 1]),
'residential_units':array([1000]),
'commercial_sqft':array([10000000]),
'industrial_sfft':array([10000000]),
'governmental_sqft':array([10000000]),
'starting_development_type_id':array([1000]),
}
development_event_history.add_elements(new_row, require_all_attributes=False)
development_event_history.flush_dataset()
gridcells = SessionConfiguration().get_dataset_from_pool('gridcell')
development_event_history = SessionConfiguration().get_dataset_from_pool('development_event_history')
unroller = UnrollGridcells()
unroller.unroll_gridcells_to_cache(gridcells, development_event_history,
cache_directory, self.base_year)
cache_directory = SimulationState().get_cache_directory()
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
gridcell = {}
for year in [1978, 1979]:
flt_directory = os.path.join(cache_directory, str(year))
gridcell[year] = DatasetFactory().get_dataset('gridcell',
package='urbansim',
subdir='datasets',
arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)}
)
self.assertEqual(gridcell[1978].get_attribute_by_id('development_type_id', changed_grid_id),
1000)
self.assertNotEqual(gridcell[1979].get_attribute_by_id('development_type_id', changed_grid_id),
gridcell[1978].get_attribute_by_id('development_type_id', changed_grid_id))
def test_gridcell_unrolling(self):
"""Checks that the unrolling of the gridcells by CacheScenarioDatabase worked correctly.
"""
cache_directory = SimulationState().get_cache_directory()
gridcells = SessionConfiguration().get_dataset_from_pool('gridcell')
development_event_history = SessionConfiguration(
).get_dataset_from_pool('development_event_history')
unroller = UnrollGridcells()
unroller.unroll_gridcells_to_cache(gridcells,
development_event_history,
cache_directory, self.base_year)
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
gridcell = {}
for year in [1976, 1977, 1979, 1980]:
#current_year = SimulationState().get_current_time()
#SimulationState().set_current_time(year)
#gridcell[year] = SessionConfiguration().get_dataset_from_pool('gridcell')
#SimulationState().set_current_time(current_year)
flt_directory = os.path.join(cache_directory, str(year))
gridcell[year] = DatasetFactory().get_dataset(
'gridcell',
package='urbansim',
subdir='datasets',
arguments={
'in_storage':
StorageFactory().get_storage(
'flt_storage', storage_location=flt_directory)
})
diff = gridcell[1980].get_attribute('residential_units') - gridcell[
1979].get_attribute('residential_units')
self.assertEqual(1, sum(diff))
diff = gridcell[1977].get_attribute('commercial_sqft') - gridcell[
1976].get_attribute('commercial_sqft')
self.assertEqual(2255 + 199 + 332 + 2785, sum(diff))
def test_input(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='fazes',
table_data={
'faz_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_sectors',
table_data={
'sector_id' :array([1,3,4]),
}
)
storage.write_table(table_name='employment_adhoc_sector_group_definitions',
table_data={
'sector_id' :array([]),
'group_id' :array([])
}
)
dc = DatasetFactory().get_dataset('faz_sector',
package='psrc_parcel',
arguments={'in_storage':storage}
)
self.assertEqual(dc.get_id_name(), ['dummy_id'])
self.assertTrue( "faz_id" in dc.get_known_attribute_names())
self.assertTrue( "sector_id" in dc.get_known_attribute_names())
self.assertEqual(dc.size(), 9)
self.assertTrue(allclose(dc.get_id_attribute(), array([
101,
103,
104,
301,
303,
304,
401,
403,
404])
))
def test_input2(self):
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="buildings",
table_data={
"building_id": array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
"zone_id": array([11, 11, 21, 21, 31, 31, 11, 11, 21, 21, 31, 31]),
"building_type_id": array([7, 99, 7, 99, 7, 99, 0, 99, 0, 7, 99, 7]),
"tenure": array([1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 1]),
},
)
ds = DatasetFactory().get_dataset("submarket", package="bayarea", arguments={"in_storage": storage})
self.assertEqual(ds.get_id_name(), ["submarket_id"])
self.assertTrue("zone_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["zone_id"], array([11, 11, 11, 21, 21, 21, 31, 31]))
self.assertTrue("building_type_id" in ds.get_known_attribute_names())
self.assertArraysEqual(ds["building_type_id"], array([0, 7, 99, 0, 7, 99, 7, 99]))
self.assertEqual(ds.size(), 8)
self.assertArraysEqual(ds.get_id_attribute(), array([11001, 11071, 11991, 21002, 21072, 21992, 31071, 31992]))
def __init__(self, file_path, package_order, test_data):
"""
file_path is the directory path to the module defining the variable
to be tested. The name of the variable to be tested is extracted
from this path. As a special case, if the file_path ends in tests/variablename.py,
we assume that this is a tests subdirectory with just a unit test, and
remove the tests/ part to find the module name.
package_order is the sequence of Opus packages in which to look for the
dataset class modules for the datasets used in this test.
test_data is a dictionary of data for the datasets to use in this test,
where the key is the name of the dataset and the values are
dictionaries containing the attribute data in numpy containers,
e.g.:
test_data={
'gridcell':{
'grid_id': array([1, 2]),
'attribute_1': array([10, 20]),
},
'household':{
'household_id': array([1, 2]),
'grid_id': array([1, 2]),
},
}
If the dataset does not its own class to be created with, it must have
an attribute 'id' which is the unique identifier of the dataset.
"""
(dirname, filename) = os.path.split(file_path)
(front, lastdir) = os.path.split(dirname)
if lastdir == 'tests':
self.file_path = os.path.join(front, filename)
else:
self.file_path = file_path
storage = StorageFactory().get_storage('dict_storage')
self.dataset_pool = DatasetPool(package_order=package_order)
for dataset_name, attribute_dict in test_data.iteritems():
storage.write_table(table_name=dataset_name,
table_data=attribute_dict)
for dataset_name, attribute_dict in test_data.iteritems(
): # the two loops are needed because
dataset = None # of possible dependencies of one dataset on another
for package in package_order:
try: # try to create a specific dataset using given packages
dataset = DatasetFactory().get_dataset(dataset_name,
package=package,
arguments={
"in_table_name":
dataset_name,
"in_storage":
storage
})
break
except:
local_resources = {
'dataset_name': dataset_name,
'in_table_name': dataset_name,
'package': package,
'in_storage': storage
#'values': attribute_dict
}
try:
dataset = DatasetFactory().get_dataset(
dataset_name,
package=package,
arguments={"resources": local_resources})
break
except:
pass
if dataset is None:
try: # try to create general dataset
dataset = Dataset(dataset_name=dataset_name,
id_name=dataset_name + "_id",
in_table_name=dataset_name,
in_storage=storage)
if dataset_name + '_id' not in dataset.get_known_attribute_names(
):
dataset = Dataset(dataset_name=dataset_name,
id_name="id",
in_table_name=dataset_name,
in_storage=storage)
except:
raise StandardError, "Error in creating dataset %s." % dataset_name
self.dataset_pool._add_dataset(dataset_name, dataset)
class TestLagVariables(opus_unittest.OpusTestCase):
def setUp(self):
self.config = TestCacheConfiguration()
self.simulation_state = SimulationState(new_instance=True)
SessionConfiguration(self.config,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = self.config['base_year']
creating_baseyear_cache_configuration = self.config[
'creating_baseyear_cache_configuration']
self.simulation_state.set_current_time(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
copytree(
os.path.join(
creating_baseyear_cache_configuration.baseyear_cache.
existing_cache_to_copy, str(self.base_year)),
os.path.join(cache_directory, str(self.base_year)))
cacher = CacheScenarioDatabase()
cacher.prepare_data_before_baseyear(
cache_directory, self.base_year,
creating_baseyear_cache_configuration)
self.config['cache_directory'] = cache_directory
cache_storage = AttributeCache().get_flt_storage_for_year(
self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
self.gridcell = DatasetFactory().get_dataset(
'gridcell',
package='urbansim',
subdir='datasets',
arguments={
'in_storage':
StorageFactory().get_storage('flt_storage',
storage_location=flt_directory)
})
def tearDown(self):
self.simulation_state.remove_singleton(delete_cache=True)
def test_lag_variables(self):
"""Test lag variables"""
# A weak test that computing a lag variable on a realistic dataset does not crash.
self.gridcell.compute_variables(
'urbansim.gridcell.n_recent_transitions_to_developed',
resources=self.config)
# The following tests are fragile, since they need to know exactly what values are being
# subtracted, and ignore any negative amount that is truncated at zero.
# If you change the "subset" dataset to a different region, you will
# have to update the expected value.
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft',
resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.commercial_sqft_lag1', resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.commercial_sqft_lag2', resources=self.config)
sqft = self.gridcell.get_attribute('commercial_sqft').sum()
sqft_lag1 = self.gridcell.get_attribute('commercial_sqft_lag1').sum()
sqft_lag2 = self.gridcell.get_attribute('commercial_sqft_lag2').sum()
logger.log_status('sqft = %s' % sqft)
logger.log_status('sqft_lag1 = %s' % sqft_lag1)
logger.log_status('sqft_lag2 = %s' % sqft_lag2)
logger.log_status('base_year = %s' % self.base_year)
self.assertEqual(self.base_year, SimulationState().get_current_time())
self.assertEqual(sqft, sqft_lag1)
self.assertEqual(578 + 2083 + 1103 + 87, sqft_lag1 - sqft_lag2)
# Do lag variables produce different results for derived attributes?
self.gridcell.compute_variables(
'urbansim.gridcell.n_recent_development_projects',
resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.n_recent_development_projects_lag1',
resources=self.config)
n_recent_projects = self.gridcell.get_attribute(
'n_recent_development_projects').sum()
n_recent_projects_lag1 = self.gridcell.get_attribute(
'n_recent_development_projects_lag1').sum()
self.assertEqual(n_recent_projects, 11)
self.assertEqual(n_recent_projects_lag1, 15)
# Do lag_variables produce different results for derived attributes without lags?
self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft',
resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.ln_commercial_sqft_lag4', resources=self.config)
sqft = self.gridcell.get_attribute('ln_commercial_sqft').sum()
sqft_lag4 = self.gridcell.get_attribute(
'ln_commercial_sqft_lag4').sum()
self.assertNotEqual(sqft, sqft_lag4)
parser.print_usage()
raise StandardError, "cache directory argument is required (-c or --cache-directory)"
if (options.start_year is None) and (options.refinements_directory is None):
parser.print_usage()
raise StandardError, "Either start year (argument -s or --start-year) or refinements directory (argument --refinements-directory) must be given."
start_year = options.start_year
end_year = options.end_year
package_order = [ package.strip() for package in options.package_order.split(",") ]
refinements = None
refinements_storage = None
if options.refinements_directory is not None:
refinements_storage = StorageFactory().get_storage('flt_storage', storage_location=options.refinements_directory)
refinements = DatasetFactory().search_for_dataset('refinement', package_order, arguments={'in_storage':refinements_storage})
years = refinements.get_attribute('year')
if start_year is None: start_year = years.min()
if end_year is None: end_year = years.max()
simulation_state = SimulationState()
simulation_state.set_cache_directory(options.cache_directory)
simulation_state.set_current_time(start_year)
attribute_cache = AttributeCache()
dataset_pool = SessionConfiguration(new_instance=True,
package_order=package_order,
in_storage=attribute_cache).get_dataset_pool()
if refinements is None:
refinements = dataset_pool.get_dataset('refinement')
years = refinements.get_attribute('year')
def prepare_for_run(self,
dataset=None,
control_totals=None,
weight_attribute=None,
excel_path=None,
excel_sheet_number=1,
excel_data_info={},
esri_storage_location=None,
dataset_name=None,
current_year=0,
datasets_and_weights_in_years=None,
dataset_pool=None):
"""
If dataset is not given, it is loaded from the esri storage. In such a case, dataset_name should be the name of the shape file (without postfix).
If control_totals is not given, it is loaded from an excel table.
datasets_and_weights_in_years is a dictionary with years as keys and tuples of dataset_name and weight attribute as values.
If its given and it has a value for the current year, its value overwrites the arguments 'dataset_name' and 'weight_attribute'.
The method returns a tuple wit three elements: dataset (Dataset object), control_totals (Dataset object), weight_attribute (string)
Those are going to be used in the run method.
"""
if control_totals is None:
logger.log_status("Getting data from Excel for AllocationModel")
excel_doc = ExcelDocument()
excel_doc.open(excel_path)
excel_doc.set_sheet(excel_sheet_number)
control_total_data = excel_doc.get_dict_table_from_column_names_and_ranges(
excel_data_info)
excel_doc.close()
excel_doc.quit()
del excel_doc
dict_storage = StorageFactory().get_storage('dict_storage')
dict_storage.write_table(table_name='control_totals',
table_data=control_total_data)
control_totals = Dataset(in_storage=dict_storage,
in_table_name='control_totals',
id_name='year')
if datasets_and_weights_in_years is not None and current_year in datasets_and_weights_in_years.keys(
):
dataset_name, weight_attribute = datasets_and_weights_in_years[
current_year]
dataset = None
if dataset is None:
logger.log_status(
"Getting data from esri_storage for AllocationModel")
esri_storage = StorageFactory().get_storage(
'esri_storage', storage_location=esri_storage_location)
#Was: dataset = Dataset(in_storage=esri_storage, in_table_name=dataset_name, id_name=dataset_name+'_id', dataset_name=dataset_name)
dataset = DatasetFactory().search_for_dataset(
dataset_name,
dataset_pool.get_package_order(),
arguments={
'in_storage': esri_storage,
'id_name': dataset_name + '_id'
})
dataset_pool.add_datasets_if_not_included({
dataset_name:
dataset,
control_totals.get_dataset_name():
control_totals
})
return (dataset, control_totals, weight_attribute)
def create_view(database, table_to_link_name, dataset_name):
df = DatasetFactory()
spatial_table_name = '%s_shp'%df._table_module_class_names_for_dataset(dataset_name)[0]
try:
spatial_table = database.get_table(spatial_table_name)
except:
print 'Error, could not create view because spatial table %s could not be found'%spatial_table_name
return
table_to_link = database.get_table(table_to_link_name)
spatial_primary_keys = database.get_primary_keys_for_table(spatial_table)
table_to_link_primary_keys = database.get_primary_keys_for_table(table_to_link)
primary_key_error_msg = 'The table %s either has no primary key or has a composite primary key. View creation does not support such tables'
if spatial_primary_keys == [] or len(spatial_primary_keys) > 1 is not None:
raise Exception(primary_key_error_msg%(spatial_table_name))
if table_to_link_primary_keys == [] or len(table_to_link_primary_keys) > 1 is not None:
raise Exception(primary_key_error_msg%(table_to_link_name))
cols_from_spatial = [c.name for c in spatial_table.c]
cols_from_linked = [c.name for c in table_to_link.c if c.name not in cols_from_spatial]
if table_to_link_primary_keys[0].name not in cols_from_spatial:
print 'WARNING: view will not be able to be created because the spatial table does not contain the column %s'%table_to_link_primary_keys[0].name
cols = ','.join(['s.%s'%c for c in cols_from_spatial] + ['l.%s'%c for c in cols_from_linked])
params = {
'join_col': table_to_link_primary_keys[0].name,
'to_link':table_to_link,
'spatial_table':spatial_table_name,
'cols':cols,
'spatial_key':table_to_link_primary_keys[0].name
}
qry = ('''
CREATE OR REPLACE VIEW %(to_link)s_view
AS SELECT %(cols)s from %(to_link)s as l, %(spatial_table)s as s
WHERE l.%(join_col)s=s.%(spatial_key)s
'''%params)
try:
database.execute(qry)
print qry
except:
print 'Error, could not create view'
print qry
import traceback
traceback.print_exc()
#if __name__ == '__main__':
# from opus_core.database_management.database_server import DatabaseServer
# from opus_core.database_management.configurations.database_server_configuration import DatabaseServerConfiguration
#
# config = DatabaseServerConfiguration(protocol='postgres')
# database_server = DatabaseServer(config)
# database = database_server.get_database('psrc')
# create_view(database = database,
# table_to_link_name = 'zone_indicator_fixed',
# dataset_name = 'zone')
def __init__(self, file_path, package_order, test_data):
"""
file_path is the directory path to the module defining the variable
to be tested. The name of the variable to be tested is extracted
from this path. As a special case, if the file_path ends in tests/variablename.py,
we assume that this is a tests subdirectory with just a unit test, and
remove the tests/ part to find the module name.
package_order is the sequence of Opus packages in which to look for the
dataset class modules for the datasets used in this test.
test_data is a dictionary of data for the datasets to use in this test,
where the key is the name of the dataset and the values are
dictionaries containing the attribute data in numpy containers,
e.g.:
test_data={
'gridcell':{
'grid_id': array([1, 2]),
'attribute_1': array([10, 20]),
},
'household':{
'household_id': array([1, 2]),
'grid_id': array([1, 2]),
},
}
If the dataset does not its own class to be created with, it must have
an attribute 'id' which is the unique identifier of the dataset.
"""
(dirname, filename) = os.path.split(file_path)
(front, lastdir) = os.path.split(dirname)
if lastdir=='tests':
self.file_path = os.path.join(front,filename)
else:
self.file_path = file_path
storage = StorageFactory().get_storage('dict_storage')
self.dataset_pool = DatasetPool(package_order=package_order)
for dataset_name, attribute_dict in test_data.iteritems():
storage.write_table(table_name=dataset_name, table_data=attribute_dict)
for dataset_name, attribute_dict in test_data.iteritems(): # the two loops are needed because
dataset = None # of possible dependencies of one dataset on another
for package in package_order:
try: # try to create a specific dataset using given packages
dataset = DatasetFactory().get_dataset(dataset_name, package=package,
arguments={"in_table_name": dataset_name,
"in_storage": storage})
break
except:
local_resources = {
'dataset_name': dataset_name,
'in_table_name': dataset_name,
'package': package,
'in_storage': storage
#'values': attribute_dict
}
try:
dataset = DatasetFactory().get_dataset(dataset_name, package=package,
arguments={"resources": local_resources})
break
except:
pass
if dataset is None:
try: # try to create general dataset
dataset = Dataset(dataset_name=dataset_name,
id_name=dataset_name+"_id",
in_table_name=dataset_name,
in_storage=storage)
if dataset_name+'_id' not in dataset.get_known_attribute_names():
dataset = Dataset(dataset_name=dataset_name,
id_name="id",
in_table_name=dataset_name,
in_storage=storage)
except:
raise StandardError, "Error in creating dataset %s." % dataset_name
self.dataset_pool._add_dataset(dataset_name, dataset)
None):
parser.print_usage()
raise StandardError, "Either start year (argument -s or --start-year) or refinements directory (argument --refinements-directory) must be given."
start_year = options.start_year
end_year = options.end_year
package_order = [
package.strip() for package in options.package_order.split(",")
]
refinements = None
refinements_storage = None
if options.refinements_directory is not None:
refinements_storage = StorageFactory().get_storage(
'flt_storage', storage_location=options.refinements_directory)
refinements = DatasetFactory().search_for_dataset(
'refinement',
package_order,
arguments={'in_storage': refinements_storage})
years = refinements.get_attribute('year')
if start_year is None: start_year = years.min()
if end_year is None: end_year = years.max()
simulation_state = SimulationState()
simulation_state.set_cache_directory(options.cache_directory)
simulation_state.set_current_time(start_year)
attribute_cache = AttributeCache()
dataset_pool = SessionConfiguration(
new_instance=True,
package_order=package_order,
in_storage=attribute_cache).get_dataset_pool()
if refinements is None:
class TestLagVariables(opus_unittest.OpusTestCase):
def setUp(self):
self.config = TestCacheConfiguration()
self.simulation_state = SimulationState(new_instance=True)
SessionConfiguration(self.config, new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = self.config['base_year']
creating_baseyear_cache_configuration = self.config['creating_baseyear_cache_configuration']
self.simulation_state.set_current_time(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
copytree(os.path.join(creating_baseyear_cache_configuration.baseyear_cache.existing_cache_to_copy,
str(self.base_year)),
os.path.join(cache_directory, str(self.base_year)))
cacher = CacheScenarioDatabase()
cacher.prepare_data_before_baseyear(cache_directory, self.base_year, creating_baseyear_cache_configuration)
self.config['cache_directory'] = cache_directory
cache_storage = AttributeCache().get_flt_storage_for_year(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
self.gridcell = DatasetFactory().get_dataset('gridcell',
package='urbansim',
subdir='datasets',
arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)}
)
def tearDown(self):
self.simulation_state.remove_singleton(delete_cache=True)
def test_lag_variables(self):
"""Test lag variables"""
# A weak test that computing a lag variable on a realistic dataset does not crash.
self.gridcell.compute_variables('urbansim.gridcell.n_recent_transitions_to_developed',
resources=self.config)
# The following tests are fragile, since they need to know exactly what values are being
# subtracted, and ignore any negative amount that is truncated at zero.
# If you change the "subset" dataset to a different region, you will
# have to update the expected value.
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft_lag1',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft_lag2',
resources=self.config)
sqft = self.gridcell.get_attribute('commercial_sqft').sum()
sqft_lag1 = self.gridcell.get_attribute('commercial_sqft_lag1').sum()
sqft_lag2 = self.gridcell.get_attribute('commercial_sqft_lag2').sum()
logger.log_status('sqft = %s' % sqft)
logger.log_status('sqft_lag1 = %s' % sqft_lag1)
logger.log_status('sqft_lag2 = %s' % sqft_lag2)
logger.log_status('base_year = %s' % self.base_year)
self.assertEqual(self.base_year, SimulationState().get_current_time())
self.assertEqual(sqft, sqft_lag1)
self.assertEqual(578+2083+1103+87, sqft_lag1 - sqft_lag2)
# Do lag variables produce different results for derived attributes?
self.gridcell.compute_variables('urbansim.gridcell.n_recent_development_projects',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.n_recent_development_projects_lag1',
resources=self.config)
n_recent_projects = self.gridcell.get_attribute('n_recent_development_projects').sum()
n_recent_projects_lag1 = self.gridcell.get_attribute('n_recent_development_projects_lag1').sum()
self.assertEqual(n_recent_projects, 11)
self.assertEqual(n_recent_projects_lag1, 15)
# Do lag_variables produce different results for derived attributes without lags?
self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft_lag4',
resources=self.config)
sqft = self.gridcell.get_attribute('ln_commercial_sqft').sum()
sqft_lag4 = self.gridcell.get_attribute('ln_commercial_sqft_lag4').sum()
self.assertNotEqual(sqft, sqft_lag4)
