def setUp(self): run_configuration = TestCacheConfiguration() SimulationState(new_instance=True) SessionConfiguration(run_configuration, new_instance=True, package_order=['urbansim', 'opus_core'], in_storage=AttributeCache()) self.base_year = run_configuration['base_year'] self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp') # Use the test cache. opus_core_path = package().get_opus_core_path() test_cache_path = os.path.join(opus_core_path, 'data', 'test_cache') new_cache_path = os.path.join(self.temp_dir, 'cache') copytree(test_cache_path, new_cache_path) # Make sure the copied files are writable. for (dirpath, dirnames, filenames) in os.walk(new_cache_path): for file_name in filenames: full_path = os.path.join(dirpath, file_name) os.chmod(full_path, S_IWRITE | S_IREAD) SimulationState().set_cache_directory(new_cache_path) SimulationState().set_current_time(self.base_year) self.config = Resources(run_configuration) cache_directory = SimulationState().get_cache_directory() self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
def get_dataset_pool(self, package_order): opus_core_path = package().get_opus_core_path() cache_dir = os.path.join(opus_core_path, 'data', 'test_cache', '1980') # These imports are here to prevent cycles in the imports. from opus_core.resources import Resources from opus_core.store.flt_storage import flt_storage storage = flt_storage(Resources({'storage_location': cache_dir})) return DatasetPool(package_order, storage=storage)
def get_dataset_pool(self, package_order): opus_core_path = package().get_opus_core_path() cache_dir = os.path.join(opus_core_path, 'data', 'test_cache', '1980') # These imports are here to prevent cycles in the imports. from opus_core.resources import Resources from opus_core.store.flt_storage import flt_storage storage = flt_storage(Resources({'storage_location':cache_dir})) return DatasetPool(package_order, storage=storage)
def setUp(self): self.config = TestCacheConfiguration() opus_core_path = package().get_opus_core_path() cache_dir = os.path.join(opus_core_path, "data", "test_cache") SimulationState(new_instance=True).set_current_time(self.config["base_year"]) SimulationState().set_cache_directory(cache_dir) SessionConfiguration( self.config, new_instance=True, package_order=["urbansim", "opus_core"], in_storage=AttributeCache() )
def setUp(self): self.config = TestCacheConfiguration() opus_core_path = package().get_opus_core_path() cache_dir = os.path.join(opus_core_path, 'data', 'test_cache') SimulationState(new_instance=True).set_current_time( self.config['base_year']) SimulationState().set_cache_directory(cache_dir) SessionConfiguration(self.config, new_instance=True, package_order=['urbansim', 'opus_core'], in_storage=AttributeCache())
def initiate_table_path(self, table_path): """self.table_path: contains list of directory to search for the xml schema Search in the directories directly in If a .../docs/database_tables exists then add it to the path. """ parent_dir_path = package().get_package_parent_path() opus_corehome = parent_dir_path dirs_in_opus_core = filter(lambda name: name.find('.') == -1, os.listdir(opus_corehome)) self.tables_path = map(lambda dir_name: join(opus_corehome, dir_name, 'docs','database_tables'), dirs_in_opus_core) self.tables_path = filter(lambda path: os.path.exists(path), self.tables_path) if table_path: self.tables_path.insert(0, table_path)
def test_change_package_order(self): """Does it find the correct dataset when there are multiple packages?""" opus_path = package().get_package_parent_path() old_sys_path = sys.path try: temp_dir = tempfile.mkdtemp(prefix='opus_tmp') # Create a temporary Opus package with a variant of the alldata dataset. temp_package_name = '__test_change_package_order__' create_package(package_parent_dir=temp_dir, package_name=temp_package_name) # Make sure Python can find this temporary package. sys.path = [temp_dir] + sys.path # Add the datasets directory. temp_package_path = os.path.join(temp_dir, temp_package_name) os.mkdir(os.path.join(temp_package_path, 'datasets')) opus_core_dataset_path = os.path.join(opus_path, 'opus_core', 'datasets') temp_package_dataset_path = os.path.join(temp_package_path, 'datasets') # Create a 'newdata' dataset. copyfile(os.path.join(opus_core_dataset_path, '__init__.py'), os.path.join(temp_package_dataset_path, '__init__.py')) copyfile( os.path.join(opus_core_dataset_path, 'alldata_dataset.py'), os.path.join(temp_package_dataset_path, 'newdata_dataset.py')) replace_string_in_files(temp_package_dataset_path, 'alldata', 'newdata') replace_string_in_files(temp_package_dataset_path, 'Alldata', 'Newdata') package_order = [temp_package_name, 'opus_core'] dataset_pool = self.get_dataset_pool(package_order) temp_dir = tempfile.mkdtemp(prefix='opus_tmp') # Create a temporary Opus package with a variant of the alldata dataset. temp_package_name = '__test_change_package_order__' create_package(package_parent_dir=temp_dir, package_name=temp_package_name) # Make sure Python can find this temporary package. sys.path = [temp_dir] + sys.path # Add the datasets directory. temp_package_path = os.path.join(temp_dir, temp_package_name) os.mkdir(os.path.join(temp_package_path, 'datasets')) opus_core_dataset_path = os.path.join(opus_path, 'opus_core', 'datasets') temp_package_dataset_path = os.path.join(temp_package_path, 'datasets') # Create a 'newdata' dataset. copyfile(os.path.join(opus_core_dataset_path, '__init__.py'), os.path.join(temp_package_dataset_path, '__init__.py')) copyfile( os.path.join(opus_core_dataset_path, 'alldata_dataset.py'), os.path.join(temp_package_dataset_path, 'newdata_dataset.py')) replace_string_in_files(temp_package_dataset_path, 'alldata', 'newdata') replace_string_in_files(temp_package_dataset_path, 'Alldata', 'Newdata') package_order = [temp_package_name, 'opus_core'] dataset_pool = self.get_dataset_pool(package_order) # Make sure it works as expected for 'alldata'. self.do_test_get_dataset(dataset_pool) # Try it for 'newdata'. newdata = dataset_pool.get_dataset('newdata') self.assert_(newdata is not None, "Could not get 'newdata' dataset") finally: sys.path = old_sys_path rmtree(temp_dir)
def test_change_package_order(self): """Does it find the correct dataset when there are multiple packages?""" opus_path = package().get_package_parent_path() old_sys_path = sys.path try: temp_dir = tempfile.mkdtemp(prefix='opus_tmp') # Create a temporary Opus package with a variant of the alldata dataset. temp_package_name = '__test_change_package_order__' create_package(package_parent_dir=temp_dir, package_name=temp_package_name) # Make sure Python can find this temporary package. sys.path = [temp_dir] + sys.path # Add the datasets directory. temp_package_path = os.path.join(temp_dir, temp_package_name) os.mkdir(os.path.join(temp_package_path, 'datasets')) opus_core_dataset_path = os.path.join(opus_path, 'opus_core', 'datasets') temp_package_dataset_path = os.path.join(temp_package_path, 'datasets') # Create a 'newdata' dataset. copyfile(os.path.join(opus_core_dataset_path, '__init__.py'), os.path.join(temp_package_dataset_path, '__init__.py')) copyfile(os.path.join(opus_core_dataset_path, 'alldata_dataset.py'), os.path.join(temp_package_dataset_path, 'newdata_dataset.py')) replace_string_in_files(temp_package_dataset_path, 'alldata', 'newdata') replace_string_in_files(temp_package_dataset_path, 'Alldata', 'Newdata') package_order = [temp_package_name, 'opus_core'] dataset_pool = self.get_dataset_pool(package_order) temp_dir = tempfile.mkdtemp(prefix='opus_tmp') # Create a temporary Opus package with a variant of the alldata dataset. temp_package_name = '__test_change_package_order__' create_package(package_parent_dir=temp_dir, package_name=temp_package_name) # Make sure Python can find this temporary package. sys.path = [temp_dir] + sys.path # Add the datasets directory. temp_package_path = os.path.join(temp_dir, temp_package_name) os.mkdir(os.path.join(temp_package_path, 'datasets')) opus_core_dataset_path = os.path.join(opus_path, 'opus_core', 'datasets') temp_package_dataset_path = os.path.join(temp_package_path, 'datasets') # Create a 'newdata' dataset. copyfile(os.path.join(opus_core_dataset_path, '__init__.py'), os.path.join(temp_package_dataset_path, '__init__.py')) copyfile(os.path.join(opus_core_dataset_path, 'alldata_dataset.py'), os.path.join(temp_package_dataset_path, 'newdata_dataset.py')) replace_string_in_files(temp_package_dataset_path, 'alldata', 'newdata') replace_string_in_files(temp_package_dataset_path, 'Alldata', 'Newdata') package_order = [temp_package_name, 'opus_core'] dataset_pool = self.get_dataset_pool(package_order) # Make sure it works as expected for 'alldata'. self.do_test_get_dataset(dataset_pool) # Try it for 'newdata'. newdata = dataset_pool.get_dataset('newdata') self.assert_(newdata is not None, "Could not get 'newdata' dataset") finally: sys.path = old_sys_path rmtree(temp_dir)
def get_dataset_pool(self, package_order=["opus_core"]): opus_core_path = package().get_opus_core_path() cache_dir = os.path.join(opus_core_path, 'data', 'test_cache', '1980') storage = StorageFactory().get_storage('flt_storage', storage_location = cache_dir) return DatasetPool(package_order, storage=storage)
def __init__(self): from copy import deepcopy config = deepcopy(AbstractUrbansimConfiguration()) cache_directory = os.path.join(package().get_opus_core_path(), 'data', 'test_cache') config_changes = { 'description': 'Test cache', 'base_year': 1980, 'years': (1981, 1982), 'models': [ 'prescheduled_events', 'events_coordinator', 'residential_land_share_model', 'land_price_model', 'development_project_transition_model', 'residential_development_project_location_choice_model', 'commercial_development_project_location_choice_model', 'industrial_development_project_location_choice_model', 'development_event_transition_model', 'events_coordinator', 'residential_land_share_model', 'household_transition_model', 'employment_transition_model', 'household_relocation_model', 'household_location_choice_model', 'employment_relocation_model', { 'employment_location_choice_model': { 'group_members': '_all_' } }, 'distribute_unplaced_jobs_model', ], 'scenario_database_configuration': ScenarioDatabaseConfiguration( database_name='eugene_1980_baseyear', ), 'cache_directory': cache_directory, 'creating_baseyear_cache_configuration': CreatingBaseyearCacheConfiguration( cache_directory_root='c:/urbansim_cache', cache_from_database=False, baseyear_cache=BaseyearCacheConfiguration( existing_cache_to_copy=cache_directory, ), cache_scenario_database= 'urbansim.model_coordinators.cache_scenario_database', tables_to_cache=[ 'annual_employment_control_totals', 'annual_household_control_totals', 'buildings', 'building_types', 'development_event_history', 'gridcells', 'households', 'job_building_types', 'jobs', 'travel_data', 'zones', 'counties', 'commercial_development_location_choice_model_coefficients', 'commercial_development_location_choice_model_specification', 'commercial_employment_location_choice_model_coefficients', 'commercial_employment_location_choice_model_specification', 'home_based_employment_location_choice_model_specification', 'home_based_employment_location_choice_model_coefficients', 'industrial_employment_location_choice_model_coefficients', 'industrial_employment_location_choice_model_specification', 'industrial_development_location_choice_model_coefficients', 'industrial_development_location_choice_model_specification', 'residential_development_location_choice_model_coefficients', 'residential_development_location_choice_model_specification', #'fazes', 'urbansim_constants', 'household_location_choice_model_coefficients', 'household_location_choice_model_specification', 'land_price_model_coefficients', 'land_price_model_specification', 'residential_land_share_model_coefficients', 'residential_land_share_model_specification', #'plan_type_group_definitions', #'plan_type_groups', #'large_areas', 'household_characteristics_for_ht', 'development_types', 'development_type_group_definitions', 'development_constraints', 'annual_relocation_rates_for_households', 'annual_relocation_rates_for_jobs', 'base_year', 'cities', #'development_events', 'development_type_groups', 'employment_adhoc_sector_group_definitions', 'employment_adhoc_sector_groups', 'employment_sectors', 'plan_types', 'race_names', 'target_vacancies' #'jobs_for_estimation', #'households_for_estimation', #'development_events_exogenous', ], tables_to_cache_nchunks={'gridcells': 1}, tables_to_copy_to_previous_years={ 'development_type_group_definitions': 1975, 'development_type_groups': 1975, 'development_types': 1975, 'urbansim_constants': 1975, }, ), } config.merge(config_changes) self.merge(config)
def setUp(self): opus_core_path = package().get_opus_core_path() cache_dir = os.path.join(opus_core_path, 'data', 'test_cache') self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp') run_dir = os.path.join(self.temp_dir, 'tests', 'runs') os.makedirs(run_dir) self.config = { 'models': ['my_regression_model', 'my_meta_model'], 'years': (1981, 1982), 'base_year': 1980, 'cache_directory': run_dir, 'models_configuration': { 'aging_model': { 'controller': { 'import': {'opus_core.simple_model': 'SimpleModel'}, 'init': {'name': 'SimpleModel'}, 'run': { 'arguments': { 'dataset': 'household', 'expression': "'household.age_of_head + 10'", 'outcome_attribute': "'age_of_head'", 'dataset_pool': 'dataset_pool' } } } }, 'my_meta_model': { 'controller': { 'import': {'opus_core.iterative_meta_model': 'IterativeMetaModel'}, 'init': {'arguments': {'models': ['aging_model'], 'configuration': 'resources', 'datasets_to_preload': ['household']}, 'name': 'IterativeMetaModel'}, 'run': {'arguments': { 'year': 'year', #'condition': "'alldata.aggregate_all(numpy.logical_not(household.age_of_head > 70)) == 0'", 'condition': "'household.age_of_head > 70'", } } } }, 'my_regression_model': { 'controller': { 'import': {'opus_core.regression_model': 'RegressionModel'}, 'init': {'arguments': {'dataset_pool': 'dataset_pool'}, 'name': 'RegressionModel'}, 'prepare_for_run': { 'arguments': { 'coefficients_storage': 'base_cache_storage', 'coefficients_table': "'land_price_model_coefficients'", 'specification_storage': 'base_cache_storage', 'specification_table': "'land_price_model_specification'", }, 'name': 'prepare_for_run', 'output': '(_specification, _coefficients, dummy)' }, 'run': { 'arguments': { 'coefficients': '_coefficients', 'specification': '_specification', 'dataset': 'gridcell' } } } }, }, 'datasets_to_preload': {'gridcell': {}}, 'creating_baseyear_cache_configuration': CreatingBaseyearCacheConfiguration( cache_directory_root = run_dir, cache_from_database = False, baseyear_cache = BaseyearCacheConfiguration(years_to_cache = [1980], existing_cache_to_copy = cache_dir) ), 'dataset_pool_configuration': DatasetPoolConfiguration( package_order=['urbansim', 'opus_core'], ), 'model_system':'urbansim.model_coordinators.model_system', } CacheFltData().run(self.config)
def __init__(self): from copy import deepcopy config = deepcopy(AbstractUrbansimConfiguration()) cache_directory = os.path.join(package().get_opus_core_path(), 'data', 'test_cache') config_changes = { 'description':'Test cache', 'base_year':1980, 'years':(1981, 1982), 'models': [ 'prescheduled_events', 'events_coordinator', 'residential_land_share_model', 'land_price_model', 'development_project_transition_model', 'residential_development_project_location_choice_model', 'commercial_development_project_location_choice_model', 'industrial_development_project_location_choice_model', 'development_event_transition_model', 'events_coordinator', 'residential_land_share_model', 'household_transition_model', 'employment_transition_model', 'household_relocation_model', 'household_location_choice_model', 'employment_relocation_model', { 'employment_location_choice_model': { 'group_members': '_all_'}}, 'distribute_unplaced_jobs_model', ], 'scenario_database_configuration': ScenarioDatabaseConfiguration( database_name = 'eugene_1980_baseyear', ), 'cache_directory': cache_directory, 'creating_baseyear_cache_configuration':CreatingBaseyearCacheConfiguration( cache_directory_root = 'c:/urbansim_cache', cache_from_database = False, baseyear_cache = BaseyearCacheConfiguration( existing_cache_to_copy = cache_directory, ), cache_scenario_database = 'urbansim.model_coordinators.cache_scenario_database', tables_to_cache = [ 'annual_employment_control_totals', 'annual_household_control_totals', 'buildings', 'building_types', 'development_event_history', 'gridcells', 'households', 'job_building_types', 'jobs', 'travel_data', 'zones', 'counties', 'commercial_development_location_choice_model_coefficients', 'commercial_development_location_choice_model_specification', 'commercial_employment_location_choice_model_coefficients', 'commercial_employment_location_choice_model_specification', 'home_based_employment_location_choice_model_specification', 'home_based_employment_location_choice_model_coefficients', 'industrial_employment_location_choice_model_coefficients', 'industrial_employment_location_choice_model_specification', 'industrial_development_location_choice_model_coefficients', 'industrial_development_location_choice_model_specification', 'residential_development_location_choice_model_coefficients', 'residential_development_location_choice_model_specification', #'fazes', 'urbansim_constants', 'household_location_choice_model_coefficients', 'household_location_choice_model_specification', 'land_price_model_coefficients', 'land_price_model_specification', 'residential_land_share_model_coefficients', 'residential_land_share_model_specification', #'plan_type_group_definitions', #'plan_type_groups', #'large_areas', 'household_characteristics_for_ht', 'development_types', 'development_type_group_definitions', 'development_constraints', 'annual_relocation_rates_for_households', 'annual_relocation_rates_for_jobs', 'base_year', 'cities', #'development_events', 'development_type_groups', 'employment_adhoc_sector_group_definitions', 'employment_adhoc_sector_groups', 'employment_sectors', 'plan_types', 'race_names', 'target_vacancies' #'jobs_for_estimation', #'households_for_estimation', #'development_events_exogenous', ], tables_to_cache_nchunks = {'gridcells': 1}, tables_to_copy_to_previous_years = { 'development_type_group_definitions': 1975, 'development_type_groups': 1975, 'development_types': 1975, 'urbansim_constants': 1975, }, ), } config.merge(config_changes) self.merge(config)