def setUp(self):
run_configuration = TestCacheConfiguration()
SimulationState(new_instance=True)
SessionConfiguration(run_configuration, new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = run_configuration['base_year']
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Use the test cache.
opus_core_path = package().get_opus_core_path()
test_cache_path = os.path.join(opus_core_path, 'data', 'test_cache')
new_cache_path = os.path.join(self.temp_dir, 'cache')
copytree(test_cache_path, new_cache_path)
# Make sure the copied files are writable.
for (dirpath, dirnames, filenames) in os.walk(new_cache_path):
for file_name in filenames:
full_path = os.path.join(dirpath, file_name)
os.chmod(full_path, S_IWRITE | S_IREAD)
SimulationState().set_cache_directory(new_cache_path)
SimulationState().set_current_time(self.base_year)
self.config = Resources(run_configuration)
cache_directory = SimulationState().get_cache_directory()
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
def setUp(self):
run_configuration = TestCacheConfiguration()
SimulationState(new_instance=True)
SessionConfiguration(run_configuration,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = run_configuration['base_year']
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Use the test cache.
opus_core_path = package().get_opus_core_path()
test_cache_path = os.path.join(opus_core_path, 'data', 'test_cache')
new_cache_path = os.path.join(self.temp_dir, 'cache')
copytree(test_cache_path, new_cache_path)
# Make sure the copied files are writable.
for (dirpath, dirnames, filenames) in os.walk(new_cache_path):
for file_name in filenames:
full_path = os.path.join(dirpath, file_name)
os.chmod(full_path, S_IWRITE | S_IREAD)
SimulationState().set_cache_directory(new_cache_path)
SimulationState().set_current_time(self.base_year)
self.config = Resources(run_configuration)
cache_directory = SimulationState().get_cache_directory()
self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
def get_dataset_pool(self, package_order):
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, 'data', 'test_cache', '1980')
# These imports are here to prevent cycles in the imports.
from opus_core.resources import Resources
from opus_core.store.flt_storage import flt_storage
storage = flt_storage(Resources({'storage_location': cache_dir}))
return DatasetPool(package_order, storage=storage)
def get_dataset_pool(self, package_order):
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, 'data', 'test_cache', '1980')
# These imports are here to prevent cycles in the imports.
from opus_core.resources import Resources
from opus_core.store.flt_storage import flt_storage
storage = flt_storage(Resources({'storage_location':cache_dir}))
return DatasetPool(package_order, storage=storage)
def setUp(self):
self.config = TestCacheConfiguration()
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, "data", "test_cache")
SimulationState(new_instance=True).set_current_time(self.config["base_year"])
SimulationState().set_cache_directory(cache_dir)
SessionConfiguration(
self.config, new_instance=True, package_order=["urbansim", "opus_core"], in_storage=AttributeCache()
)
def setUp(self):
self.config = TestCacheConfiguration()
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, 'data', 'test_cache')
SimulationState(new_instance=True).set_current_time(
self.config['base_year'])
SimulationState().set_cache_directory(cache_dir)
SessionConfiguration(self.config,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
def initiate_table_path(self, table_path):
"""self.table_path: contains list of directory to search for the xml schema
Search in the directories directly in If a .../docs/database_tables exists then add it to the path. """
parent_dir_path = package().get_package_parent_path()
opus_corehome = parent_dir_path
dirs_in_opus_core = filter(lambda name: name.find('.') == -1, os.listdir(opus_corehome))
self.tables_path = map(lambda dir_name: join(opus_corehome, dir_name, 'docs','database_tables'), dirs_in_opus_core)
self.tables_path = filter(lambda path: os.path.exists(path), self.tables_path)
if table_path:
self.tables_path.insert(0, table_path)
def initiate_table_path(self, table_path):
"""self.table_path: contains list of directory to search for the xml schema
Search in the directories directly in If a .../docs/database_tables exists then add it to the path. """
parent_dir_path = package().get_package_parent_path()
opus_corehome = parent_dir_path
dirs_in_opus_core = filter(lambda name: name.find('.') == -1, os.listdir(opus_corehome))
self.tables_path = map(lambda dir_name: join(opus_corehome, dir_name, 'docs','database_tables'), dirs_in_opus_core)
self.tables_path = filter(lambda path: os.path.exists(path), self.tables_path)
if table_path:
self.tables_path.insert(0, table_path)
def test_change_package_order(self):
"""Does it find the correct dataset when there are multiple packages?"""
opus_path = package().get_package_parent_path()
old_sys_path = sys.path
try:
temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Create a temporary Opus package with a variant of the alldata dataset.
temp_package_name = '__test_change_package_order__'
create_package(package_parent_dir=temp_dir,
package_name=temp_package_name)
# Make sure Python can find this temporary package.
sys.path = [temp_dir] + sys.path
# Add the datasets directory.
temp_package_path = os.path.join(temp_dir, temp_package_name)
os.mkdir(os.path.join(temp_package_path, 'datasets'))
opus_core_dataset_path = os.path.join(opus_path, 'opus_core',
'datasets')
temp_package_dataset_path = os.path.join(temp_package_path,
'datasets')
# Create a 'newdata' dataset.
copyfile(os.path.join(opus_core_dataset_path, '__init__.py'),
os.path.join(temp_package_dataset_path, '__init__.py'))
copyfile(
os.path.join(opus_core_dataset_path, 'alldata_dataset.py'),
os.path.join(temp_package_dataset_path, 'newdata_dataset.py'))
replace_string_in_files(temp_package_dataset_path, 'alldata',
'newdata')
replace_string_in_files(temp_package_dataset_path, 'Alldata',
'Newdata')
package_order = [temp_package_name, 'opus_core']
dataset_pool = self.get_dataset_pool(package_order)
temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Create a temporary Opus package with a variant of the alldata dataset.
temp_package_name = '__test_change_package_order__'
create_package(package_parent_dir=temp_dir,
package_name=temp_package_name)
# Make sure Python can find this temporary package.
sys.path = [temp_dir] + sys.path
# Add the datasets directory.
temp_package_path = os.path.join(temp_dir, temp_package_name)
os.mkdir(os.path.join(temp_package_path, 'datasets'))
opus_core_dataset_path = os.path.join(opus_path, 'opus_core',
'datasets')
temp_package_dataset_path = os.path.join(temp_package_path,
'datasets')
# Create a 'newdata' dataset.
copyfile(os.path.join(opus_core_dataset_path, '__init__.py'),
os.path.join(temp_package_dataset_path, '__init__.py'))
copyfile(
os.path.join(opus_core_dataset_path, 'alldata_dataset.py'),
os.path.join(temp_package_dataset_path, 'newdata_dataset.py'))
replace_string_in_files(temp_package_dataset_path, 'alldata',
'newdata')
replace_string_in_files(temp_package_dataset_path, 'Alldata',
'Newdata')
package_order = [temp_package_name, 'opus_core']
dataset_pool = self.get_dataset_pool(package_order)
# Make sure it works as expected for 'alldata'.
self.do_test_get_dataset(dataset_pool)
# Try it for 'newdata'.
newdata = dataset_pool.get_dataset('newdata')
self.assert_(newdata is not None,
"Could not get 'newdata' dataset")
finally:
sys.path = old_sys_path
rmtree(temp_dir)
def test_change_package_order(self):
"""Does it find the correct dataset when there are multiple packages?"""
opus_path = package().get_package_parent_path()
old_sys_path = sys.path
try:
temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Create a temporary Opus package with a variant of the alldata dataset.
temp_package_name = '__test_change_package_order__'
create_package(package_parent_dir=temp_dir, package_name=temp_package_name)
# Make sure Python can find this temporary package.
sys.path = [temp_dir] + sys.path
# Add the datasets directory.
temp_package_path = os.path.join(temp_dir, temp_package_name)
os.mkdir(os.path.join(temp_package_path, 'datasets'))
opus_core_dataset_path = os.path.join(opus_path, 'opus_core', 'datasets')
temp_package_dataset_path = os.path.join(temp_package_path, 'datasets')
# Create a 'newdata' dataset.
copyfile(os.path.join(opus_core_dataset_path, '__init__.py'),
os.path.join(temp_package_dataset_path, '__init__.py'))
copyfile(os.path.join(opus_core_dataset_path, 'alldata_dataset.py'),
os.path.join(temp_package_dataset_path, 'newdata_dataset.py'))
replace_string_in_files(temp_package_dataset_path, 'alldata', 'newdata')
replace_string_in_files(temp_package_dataset_path, 'Alldata', 'Newdata')
package_order = [temp_package_name, 'opus_core']
dataset_pool = self.get_dataset_pool(package_order)
temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
# Create a temporary Opus package with a variant of the alldata dataset.
temp_package_name = '__test_change_package_order__'
create_package(package_parent_dir=temp_dir, package_name=temp_package_name)
# Make sure Python can find this temporary package.
sys.path = [temp_dir] + sys.path
# Add the datasets directory.
temp_package_path = os.path.join(temp_dir, temp_package_name)
os.mkdir(os.path.join(temp_package_path, 'datasets'))
opus_core_dataset_path = os.path.join(opus_path, 'opus_core', 'datasets')
temp_package_dataset_path = os.path.join(temp_package_path, 'datasets')
# Create a 'newdata' dataset.
copyfile(os.path.join(opus_core_dataset_path, '__init__.py'),
os.path.join(temp_package_dataset_path, '__init__.py'))
copyfile(os.path.join(opus_core_dataset_path, 'alldata_dataset.py'),
os.path.join(temp_package_dataset_path, 'newdata_dataset.py'))
replace_string_in_files(temp_package_dataset_path, 'alldata', 'newdata')
replace_string_in_files(temp_package_dataset_path, 'Alldata', 'Newdata')
package_order = [temp_package_name, 'opus_core']
dataset_pool = self.get_dataset_pool(package_order)
# Make sure it works as expected for 'alldata'.
self.do_test_get_dataset(dataset_pool)
# Try it for 'newdata'.
newdata = dataset_pool.get_dataset('newdata')
self.assert_(newdata is not None, "Could not get 'newdata' dataset")
finally:
sys.path = old_sys_path
rmtree(temp_dir)
def get_dataset_pool(self, package_order=["opus_core"]):
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, 'data', 'test_cache', '1980')
storage = StorageFactory().get_storage('flt_storage', storage_location = cache_dir)
return DatasetPool(package_order, storage=storage)
def __init__(self):
from copy import deepcopy
config = deepcopy(AbstractUrbansimConfiguration())
cache_directory = os.path.join(package().get_opus_core_path(), 'data',
'test_cache')
config_changes = {
'description':
'Test cache',
'base_year':
1980,
'years': (1981, 1982),
'models': [
'prescheduled_events',
'events_coordinator',
'residential_land_share_model',
'land_price_model',
'development_project_transition_model',
'residential_development_project_location_choice_model',
'commercial_development_project_location_choice_model',
'industrial_development_project_location_choice_model',
'development_event_transition_model',
'events_coordinator',
'residential_land_share_model',
'household_transition_model',
'employment_transition_model',
'household_relocation_model',
'household_location_choice_model',
'employment_relocation_model',
{
'employment_location_choice_model': {
'group_members': '_all_'
}
},
'distribute_unplaced_jobs_model',
],
'scenario_database_configuration':
ScenarioDatabaseConfiguration(
database_name='eugene_1980_baseyear', ),
'cache_directory':
cache_directory,
'creating_baseyear_cache_configuration':
CreatingBaseyearCacheConfiguration(
cache_directory_root='c:/urbansim_cache',
cache_from_database=False,
baseyear_cache=BaseyearCacheConfiguration(
existing_cache_to_copy=cache_directory, ),
cache_scenario_database=
'urbansim.model_coordinators.cache_scenario_database',
tables_to_cache=[
'annual_employment_control_totals',
'annual_household_control_totals',
'buildings',
'building_types',
'development_event_history',
'gridcells',
'households',
'job_building_types',
'jobs',
'travel_data',
'zones',
'counties',
'commercial_development_location_choice_model_coefficients',
'commercial_development_location_choice_model_specification',
'commercial_employment_location_choice_model_coefficients',
'commercial_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_specification',
'industrial_development_location_choice_model_coefficients',
'industrial_development_location_choice_model_specification',
'residential_development_location_choice_model_coefficients',
'residential_development_location_choice_model_specification',
#'fazes',
'urbansim_constants',
'household_location_choice_model_coefficients',
'household_location_choice_model_specification',
'land_price_model_coefficients',
'land_price_model_specification',
'residential_land_share_model_coefficients',
'residential_land_share_model_specification',
#'plan_type_group_definitions',
#'plan_type_groups',
#'large_areas',
'household_characteristics_for_ht',
'development_types',
'development_type_group_definitions',
'development_constraints',
'annual_relocation_rates_for_households',
'annual_relocation_rates_for_jobs',
'base_year',
'cities',
#'development_events',
'development_type_groups',
'employment_adhoc_sector_group_definitions',
'employment_adhoc_sector_groups',
'employment_sectors',
'plan_types',
'race_names',
'target_vacancies'
#'jobs_for_estimation',
#'households_for_estimation',
#'development_events_exogenous',
],
tables_to_cache_nchunks={'gridcells': 1},
tables_to_copy_to_previous_years={
'development_type_group_definitions': 1975,
'development_type_groups': 1975,
'development_types': 1975,
'urbansim_constants': 1975,
},
),
}
config.merge(config_changes)
self.merge(config)
def setUp(self):
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, 'data', 'test_cache')
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
run_dir = os.path.join(self.temp_dir, 'tests', 'runs')
os.makedirs(run_dir)
self.config = {
'models': ['my_regression_model', 'my_meta_model'],
'years': (1981, 1982),
'base_year': 1980,
'cache_directory': run_dir,
'models_configuration': {
'aging_model': {
'controller': {
'import': {'opus_core.simple_model': 'SimpleModel'},
'init': {'name': 'SimpleModel'},
'run': {
'arguments': {
'dataset': 'household',
'expression': "'household.age_of_head + 10'",
'outcome_attribute': "'age_of_head'",
'dataset_pool': 'dataset_pool'
}
}
}
},
'my_meta_model': {
'controller': {
'import': {'opus_core.iterative_meta_model': 'IterativeMetaModel'},
'init': {'arguments': {'models': ['aging_model'],
'configuration': 'resources',
'datasets_to_preload': ['household']},
'name': 'IterativeMetaModel'},
'run': {'arguments': {
'year': 'year',
#'condition': "'alldata.aggregate_all(numpy.logical_not(household.age_of_head > 70)) == 0'",
'condition': "'household.age_of_head > 70'",
}
}
}
},
'my_regression_model': {
'controller': {
'import': {'opus_core.regression_model': 'RegressionModel'},
'init': {'arguments': {'dataset_pool': 'dataset_pool'},
'name': 'RegressionModel'},
'prepare_for_run': {
'arguments': {
'coefficients_storage': 'base_cache_storage',
'coefficients_table': "'land_price_model_coefficients'",
'specification_storage': 'base_cache_storage',
'specification_table': "'land_price_model_specification'",
},
'name': 'prepare_for_run',
'output': '(_specification, _coefficients, dummy)'
},
'run': {
'arguments': {
'coefficients': '_coefficients',
'specification': '_specification',
'dataset': 'gridcell'
}
}
}
},
},
'datasets_to_preload': {'gridcell': {}},
'creating_baseyear_cache_configuration': CreatingBaseyearCacheConfiguration(
cache_directory_root = run_dir,
cache_from_database = False,
baseyear_cache = BaseyearCacheConfiguration(years_to_cache = [1980],
existing_cache_to_copy = cache_dir)
),
'dataset_pool_configuration': DatasetPoolConfiguration(
package_order=['urbansim', 'opus_core'],
),
'model_system':'urbansim.model_coordinators.model_system',
}
CacheFltData().run(self.config)
def __init__(self):
from copy import deepcopy
config = deepcopy(AbstractUrbansimConfiguration())
cache_directory = os.path.join(package().get_opus_core_path(), 'data', 'test_cache')
config_changes = {
'description':'Test cache',
'base_year':1980,
'years':(1981, 1982),
'models': [
'prescheduled_events',
'events_coordinator',
'residential_land_share_model',
'land_price_model',
'development_project_transition_model',
'residential_development_project_location_choice_model',
'commercial_development_project_location_choice_model',
'industrial_development_project_location_choice_model',
'development_event_transition_model',
'events_coordinator',
'residential_land_share_model',
'household_transition_model',
'employment_transition_model',
'household_relocation_model',
'household_location_choice_model',
'employment_relocation_model',
{ 'employment_location_choice_model': { 'group_members': '_all_'}},
'distribute_unplaced_jobs_model',
],
'scenario_database_configuration': ScenarioDatabaseConfiguration(
database_name = 'eugene_1980_baseyear',
),
'cache_directory': cache_directory,
'creating_baseyear_cache_configuration':CreatingBaseyearCacheConfiguration(
cache_directory_root = 'c:/urbansim_cache',
cache_from_database = False,
baseyear_cache = BaseyearCacheConfiguration(
existing_cache_to_copy = cache_directory,
),
cache_scenario_database = 'urbansim.model_coordinators.cache_scenario_database',
tables_to_cache = [
'annual_employment_control_totals',
'annual_household_control_totals',
'buildings',
'building_types',
'development_event_history',
'gridcells',
'households',
'job_building_types',
'jobs',
'travel_data',
'zones',
'counties',
'commercial_development_location_choice_model_coefficients',
'commercial_development_location_choice_model_specification',
'commercial_employment_location_choice_model_coefficients',
'commercial_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_specification',
'home_based_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_coefficients',
'industrial_employment_location_choice_model_specification',
'industrial_development_location_choice_model_coefficients',
'industrial_development_location_choice_model_specification',
'residential_development_location_choice_model_coefficients',
'residential_development_location_choice_model_specification',
#'fazes',
'urbansim_constants',
'household_location_choice_model_coefficients',
'household_location_choice_model_specification',
'land_price_model_coefficients',
'land_price_model_specification',
'residential_land_share_model_coefficients',
'residential_land_share_model_specification',
#'plan_type_group_definitions',
#'plan_type_groups',
#'large_areas',
'household_characteristics_for_ht',
'development_types',
'development_type_group_definitions',
'development_constraints',
'annual_relocation_rates_for_households',
'annual_relocation_rates_for_jobs',
'base_year',
'cities',
#'development_events',
'development_type_groups',
'employment_adhoc_sector_group_definitions',
'employment_adhoc_sector_groups',
'employment_sectors',
'plan_types',
'race_names',
'target_vacancies'
#'jobs_for_estimation',
#'households_for_estimation',
#'development_events_exogenous',
],
tables_to_cache_nchunks = {'gridcells': 1},
tables_to_copy_to_previous_years = {
'development_type_group_definitions': 1975,
'development_type_groups': 1975,
'development_types': 1975,
'urbansim_constants': 1975,
},
),
}
config.merge(config_changes)
self.merge(config)
def setUp(self):
opus_core_path = package().get_opus_core_path()
cache_dir = os.path.join(opus_core_path, 'data', 'test_cache')
self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
run_dir = os.path.join(self.temp_dir, 'tests', 'runs')
os.makedirs(run_dir)
self.config = {
'models': ['my_regression_model', 'my_meta_model'],
'years': (1981, 1982),
'base_year': 1980,
'cache_directory': run_dir,
'models_configuration': {
'aging_model': {
'controller': {
'import': {'opus_core.simple_model': 'SimpleModel'},
'init': {'name': 'SimpleModel'},
'run': {
'arguments': {
'dataset': 'household',
'expression': "'household.age_of_head + 10'",
'outcome_attribute': "'age_of_head'",
'dataset_pool': 'dataset_pool'
}
}
}
},
'my_meta_model': {
'controller': {
'import': {'opus_core.iterative_meta_model': 'IterativeMetaModel'},
'init': {'arguments': {'models': ['aging_model'],
'configuration': 'resources',
'datasets_to_preload': ['household']},
'name': 'IterativeMetaModel'},
'run': {'arguments': {
'year': 'year',
#'condition': "'alldata.aggregate_all(numpy.logical_not(household.age_of_head > 70)) == 0'",
'condition': "'household.age_of_head > 70'",
}
}
}
},
'my_regression_model': {
'controller': {
'import': {'opus_core.regression_model': 'RegressionModel'},
'init': {'arguments': {'dataset_pool': 'dataset_pool'},
'name': 'RegressionModel'},
'prepare_for_run': {
'arguments': {
'coefficients_storage': 'base_cache_storage',
'coefficients_table': "'land_price_model_coefficients'",
'specification_storage': 'base_cache_storage',
'specification_table': "'land_price_model_specification'",
},
'name': 'prepare_for_run',
'output': '(_specification, _coefficients, dummy)'
},
'run': {
'arguments': {
'coefficients': '_coefficients',
'specification': '_specification',
'dataset': 'gridcell'
}
}
}
},
},
'datasets_to_preload': {'gridcell': {}},
'creating_baseyear_cache_configuration': CreatingBaseyearCacheConfiguration(
cache_directory_root = run_dir,
cache_from_database = False,
baseyear_cache = BaseyearCacheConfiguration(years_to_cache = [1980],
existing_cache_to_copy = cache_dir)
),
'dataset_pool_configuration': DatasetPoolConfiguration(
package_order=['urbansim', 'opus_core'],
),
'model_system':'urbansim.model_coordinators.model_system',
}
CacheFltData().run(self.config)