def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] csv_data_path = params_dict['csv_data_path'] table_name = params_dict['csv_table_name'] input_storage = csv_storage(storage_location = csv_data_path) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logCB('caching all tables...\n') lst = input_storage.get_table_names() else: lst = [table_name] for i in lst: logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory)) ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage) logCB("Successfully exported all tables.")
def _compute_variable_for_prior_year(self, dataset, full_name, time, resources=None): """Create a new dataset for this variable, compute the variable, and then return the values for this variable.""" calling_dataset_pool = SessionConfiguration().get_dataset_pool() calling_time = SimulationState().get_current_time() SimulationState().set_current_time(time) try: # Get an empty dataset pool with same search paths. my_dataset_pool = DatasetPool( package_order=calling_dataset_pool.get_package_order(), storage=AttributeCache()) ds = dataset.empty_dataset_like_me(in_storage=AttributeCache()) # Don't pass any datasets via resources, since they may be from a different time. my_resources = Resources(resources) for key in my_resources: if isinstance(key, Dataset): del my_resources[key] ds.compute_variables(full_name, my_dataset_pool, resources=my_resources) values = ds.get_attribute(full_name) return values finally: SimulationState().set_current_time(calling_time)
def save_results(self, out_storage=None, model_name=None): if self.specification is None or self.coefficients is None: raise ValueError, "model specification or coefficient is None" #invalid = self.coefficients.is_invalid() if False: logger.log_warning('Invalid coefficients. Not saving results!') return if model_name is None: model_name = self.config.get('model_name_for_coefficients', None) if model_name is None: if self.model_name is not None: model_name = self.model_name else: raise ValueError, "model_name unspecified" out_storage_available = True if out_storage: pass elif 'estimation_database_configuration' in self.config: try: db_server = DatabaseServer(self.config['estimation_database_configuration']) database_name = self.config["estimation_database_configuration"].database_name if not db_server.has_database(database_name): db_server.create_database(database_name) output_db = db_server.get_database(database_name) out_storage = StorageFactory().get_storage( type='sql_storage', storage_location=output_db) except: logger.log_warning("Problem with connecting database given by 'estimation_database_configuration'.") out_storage_available = False else: logger.log_warning("No estimation_database_configuration given.") out_storage_available = False # the original model name of development_project_lcm is too long as a mysql db table name, truncate it if model_name.rfind("_development_project_location_choice_model") >=0: model_name = model_name.replace('_project', '') specification_table = '%s_specification' % model_name coefficients_table = '%s_coefficients' % model_name if out_storage_available: logger.start_block("Writing specification and coefficients into storage given by 'estimation_database_configuration'") self.specification.write(out_storage=out_storage, out_table_name=specification_table) self.coefficients.write(out_storage=out_storage, out_table_name=coefficients_table) logger.end_block() logger.start_block("Writing specification and coefficients into %s" % AttributeCache().get_storage_location()) self.specification.write(out_storage=AttributeCache(), out_table_name=specification_table) self.coefficients.write(out_storage=AttributeCache(), out_table_name=coefficients_table) logger.end_block()
def opusRun(progressCB, logCB, params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) opus_data_directory = params_dict['opus_data_directory'] opus_data_directory = paths.prepend_opus_home_if_relative( opus_data_directory) opus_data_year = params_dict['opus_data_year'] database_name = params_dict['database_name'] table_name = params_dict['table_name'] database_server_connection = params_dict['database_server_connection'] overwrite = params_dict['overwrite'] dbs_config = DatabaseServerConfiguration( database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration=dbs_config) opusdb = server.get_database(database_name=database_name, create_if_doesnt_exist=False) input_storage = sql_storage(storage_location=opusdb) attribute_cache = AttributeCache(cache_directory=opus_data_directory) output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) SimulationState().set_current_time(opus_data_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': lst = input_storage.get_table_names() else: lst = re.split(' +', table_name.strip()) tables = len(lst) lst_out = create_list_string(lst, ', ') logCB('caching tables:\n%s\n' % lst_out) for j, i in enumerate(lst, start=1): logCB("Exporting table '%s' to year %s of cache located at %s...\n" % (i, opus_data_year, opus_data_directory)) ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage, out_storage=output_storage, overwrite=overwrite, ) progressCB(100 * j / tables) logCB('successfully cached tables:\n%s\n' % lst_out)
def opusRun(progressCB,logCB,params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) # Output csv data path csv_data_path = params_dict['csv_data_path'] # Data clasification - Database (must be specified) opus_data_directory = params_dict['opus_data_directory'] # Data clasification - Dataset (explicit or ALL) opus_data_year = params_dict['opus_data_year'] # Data clasification - Array (explicit or ALL) opus_table_name = params_dict['opus_table_name'] execute_after_export = params_dict['execute_after_export'] attribute_cache = AttributeCache(cache_directory=opus_data_directory) attribute_cache_years = [int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4] if opus_data_year != 'ALL': attribute_cache_years = [opus_data_year] for year in attribute_cache_years: input_storage = attribute_cache.get_flt_storage_for_year(year) output_storage = csv_storage(storage_location = csv_data_path) SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if opus_table_name != 'ALL': opus_table_name_list = [opus_table_name] else: opus_table_name_list = input_storage.get_table_names() for i in opus_table_name_list: logCB("Exporting %s, %s, %s\n" % (i,year,opus_data_directory)) ExportStorage().export_dataset( dataset_name = i, in_storage = input_storage, out_storage = output_storage, ) logCB("Successfully exported all datasets.") if execute_after_export: file_name_list = [output_storage._get_file_path_for_table(i) for i in opus_table_name_list] subprocess.Popen([execute_after_export] + file_name_list)
def opusRun(progressCB, logCB, params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) database_name = params_dict['database_name'] opus_data_directory = params_dict['opus_data_directory'] opus_data_year = params_dict['opus_data_year'] opus_table_name = params_dict['opus_table_name'] database_server_connection = params_dict['database_server_connection'] dbs_config = DatabaseServerConfiguration( database_configuration=database_server_connection) server = DatabaseServer(database_server_configuration=dbs_config) opusdb = server.get_database(database_name=database_name) attribute_cache = AttributeCache(cache_directory=opus_data_directory) attribute_cache_years = [ int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4 ] if opus_data_year != 'ALL': attribute_cache_years = [opus_data_year] for year in attribute_cache_years: #input_storage = sql_storage(storage_location = opusdb) input_storage = attribute_cache.get_flt_storage_for_year(year) #output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year) if opus_data_year == 'ALL': opusdb = server.get_database(database_name=database_name + "_" + str(year)) output_storage = sql_storage(storage_location=opusdb) SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if opus_table_name != 'ALL': opus_table_name_list = re.split(' +', opus_table_name.strip()) else: opus_table_name_list = input_storage.get_table_names() for i in opus_table_name_list: logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory)) ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage, out_storage=output_storage, )
def opusRun(progressCB, logCB, params): params_dict = {} for key, val in params.iteritems(): params_dict[str(key)] = str(val) # Output esri data path esri_data_path = params_dict['esri_data_path'] # Data clasification - Database (must be specified) opus_data_directory = params_dict['opus_data_directory'] # Data clasification - Dataset (explicit or ALL) opus_data_year = params_dict['opus_data_year'] # Data clasification - Array (explicit or ALL) opus_table_name = params_dict['opus_table_name'] attribute_cache = AttributeCache(cache_directory=opus_data_directory) attribute_cache_years = [ int(year) for year in os.listdir(opus_data_directory) if year.isdigit() and len(year) == 4 ] if opus_data_year != 'ALL': attribute_cache_years = [opus_data_year] for year in attribute_cache_years: input_storage = attribute_cache.get_flt_storage_for_year(year) if esri_is_avail: output_storage = esri_storage(storage_location=esri_data_path) else: output_storage = None SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if opus_table_name != 'ALL': opus_table_name_list = [opus_table_name] else: opus_table_name_list = input_storage.get_table_names() for i in opus_table_name_list: logCB("Exporting %s, %s, %s\n" % (i, year, opus_data_directory)) ExportStorage().export_dataset( dataset_name=i, in_storage=input_storage, out_storage=output_storage, )
def setUp(self): run_configuration = TestCacheConfiguration() SimulationState(new_instance=True) SessionConfiguration(run_configuration, new_instance=True, package_order=['urbansim', 'opus_core'], in_storage=AttributeCache()) self.base_year = run_configuration['base_year'] self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp') # Use the test cache. opus_core_path = package().get_opus_core_path() test_cache_path = os.path.join(opus_core_path, 'data', 'test_cache') new_cache_path = os.path.join(self.temp_dir, 'cache') copytree(test_cache_path, new_cache_path) # Make sure the copied files are writable. for (dirpath, dirnames, filenames) in os.walk(new_cache_path): for file_name in filenames: full_path = os.path.join(dirpath, file_name) os.chmod(full_path, S_IWRITE | S_IREAD) SimulationState().set_cache_directory(new_cache_path) SimulationState().set_current_time(self.base_year) self.config = Resources(run_configuration) cache_directory = SimulationState().get_cache_directory() self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])
def _make_all_indicators(self, indicators, source_data): computed_indicators = {} indicators_by_dataset = {} for year in source_data.years: SimulationState().set_current_time(year) SessionConfiguration(new_instance=True, package_order=self.package_order, in_storage=AttributeCache()) for name, indicator in indicators.items(): dataset_name = indicator.dataset_name if dataset_name not in indicators_by_dataset: indicators_by_dataset[dataset_name] = [(name, indicator)] else: indicators_by_dataset[dataset_name].append( (name, indicator)) for dataset_name, indicators_in_dataset in indicators_by_dataset.items( ): dataset = SessionConfiguration().get_dataset_from_pool( dataset_name) self._make_indicators_for_dataset( dataset=dataset, indicators_in_dataset=indicators_in_dataset, source_data=source_data, computed_indicators=computed_indicators, year=year) self.computed_indicators[source_data.name] = computed_indicators return computed_indicators
def get_needed_matrices_from_emme2(self, year, cache_directory, bank_dir, matrix_variable_map, matrices_created=False): """Copies the specified emme/2 matrices into the specified travel_data variable names. """ logger.start_block('Getting matricies from emme2') try: zone_set = SessionConfiguration().get_dataset_from_pool('zone') zone_set.load_dataset() travel_data_set = self.get_travel_data_from_emme2( zone_set, bank_dir, matrix_variable_map, matrices_created) finally: logger.end_block() logger.start_block('Writing data to cache') try: next_year = year + 1 out_storage = AttributeCache().get_flt_storage_for_year(next_year) travel_data_set.write_dataset(attributes='*', out_storage=out_storage, out_table_name='travel_data') finally: logger.end_block()
def get_travel_data_from_travel_model(self, config, year, zone_set): """ """ logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...') # print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them" # print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id" # solved 3dec08 by hana input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim", "tmp") logger.log_status("input_directory: " + input_directory) in_storage = csv_storage(storage_location=input_directory) table_name = "travel_data" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) ##TODO:This may not work or may be wrong after the id_names of travel_data ##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang) existing_travel_data_set.join( travel_data_set, travel_data_set.get_non_id_primary_attribute_names(), metadata=AttributeType.PRIMARY) return existing_travel_data_set
def setUp(self): self.temp_dir = tempfile.mkdtemp(prefix='opus_tmp_test_indicators') storage = StorageFactory().get_storage('dict_storage') tables_data = { 'large_areas': { "large_area_id":array([1, 2, 3]) }, 'fazes': { "faz_id":array([1,2,3,4]), "large_area_id":array([1,2,3,3]), }, 'zones': { "zone_id":array([1,2,3,4,5]), "faz_id":array([1,2,2,3,4]), }, 'gridcells': { "grid_id":array([1,2,3,4,5,6,7,8,9]), "zone_id":array([1,1,1,2,2,3,3,4,5]), "total_land_value":array([10, 11, 12, 13, 14, 15, 16, 17, 18]), "is_in_plan_type_group_residential":array([1, 1, 1, 1, 0, 0, 1, 0, 1]) }, 'urbansim_constants':{ 'acres': array([1.5]), }, } self.year = 1000 SimulationState().set_current_time(self.year) exporter = ExportStorage() for table_name, table_data in tables_data.iteritems(): storage.write_table(table_name=table_name, table_data=table_data) exporter.export_dataset(dataset_name=table_name, in_storage=storage, out_storage=AttributeCache(cache_directory=self.temp_dir))
def test_compute_a_variable(self): """Return merged dataset for this set of years.""" test_data = { 1000: { 'tests': { 'id': array([1, 2, 3]), 'attr1': array([10, 20, 30]), }, }, 1001: { 'tests': { 'id': array([1, 2, 3]), 'attr1': array([40, 50, 60]), }, }, } cache_creator = CreateTestAttributeCache() cache_creator.create_attribute_cache_with_data(self.temp_dir, test_data) attribute_cache = AttributeCache() SessionConfiguration(new_instance=True, package_order=['opus_core'], in_storage=attribute_cache) ds = MultipleYearDatasetView( name_of_dataset_to_merge='test', in_table_name='tests', years_to_merge=[1000, 1001], attribute_cache=attribute_cache, ) ds.compute_variables(['opus_core.test.attr1_times_2'])
def prepare_for_running_macro(parser): from opus_core.file_utilities import get_resources_from_file parser.add_option("-r", "--resources", dest="resources_file_name", action="store", type="string", help="Name of file containing resources") parser.add_option("-y", "--year", dest="year", action="store", type="int", help="Year in which to 'run' the travel model") parser.add_option( "-o", "--output-file", dest="output_file", action="store", type="string", default=None, help= "Output log file. If not given, it is written into urbansim cache directory." ) (options, args) = parser.parse_args() r = get_resources_from_file(options.resources_file_name) resources = Resources(get_resources_from_file(options.resources_file_name)) SessionConfiguration( new_instance=True, package_order=resources['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) return (resources, options)
def _do_run_simple_test_run(caller, temp_dir, config, end_year=None): """Runs model system with a single model (for speed). Sets the .resources property of the caller before starting the run. """ runs_manager = RunManager(config) run_configuration = _get_run_config(temp_dir=temp_dir) insert_auto_generated_cache_directory_if_needed(run_configuration) run_configuration[ 'creating_baseyear_cache_configuration'].cache_directory_root = temp_dir run_configuration['models'] = ['land_price_model'] if end_year is not None: run_configuration['years'] = (run_configuration['years'][0], end_year) SessionConfiguration( new_instance=True, package_order=run_configuration['dataset_pool_configuration']. package_order, in_storage=AttributeCache()) insert_auto_generated_cache_directory_if_needed(run_configuration) caller.resources = run_configuration runs_manager.setup_new_run( cache_directory=run_configuration['cache_directory'], configuration=run_configuration) runs_manager.run_run(run_configuration)
def unroll_gridcells_to_cache_from_buildings(self, gridcells, buildings, cache_directory, base_year): """Populate the cache with the unrolled gridcells info derived from the buildings table. """ logger.start_block('Unrolling gridcell data from buildings') try: storage = AttributeCache().get_flt_storage_for_year(base_year) urbansim_constant = SessionConfiguration().get_dataset_from_pool('urbansim_constant') print "recent_years = %s" % urbansim_constant['recent_years'] recent_years = urbansim_constant['recent_years'] roller = RollbackGridcellsFromBuildings() for year in range(base_year, base_year-recent_years-1, -1): logger.start_block('Unrolling gridcells into year %d' % (year-1)) try: roller.unroll_gridcells_for_one_year(gridcells, buildings, year, dataset_pool=SessionConfiguration().get_dataset_pool()) flt_directory = os.path.join(cache_directory, str(year-1)) flt_storage = StorageFactory().get_storage( type='flt_storage', subdir='store', storage_location=flt_directory) gridcells.write_dataset(out_storage=flt_storage) finally: logger.end_block() finally: logger.end_block()
def test_at_year_2000(self): cache_dir = os.path.join(self.temp_dir, 'cache') data = { self._id_name: array([1, 2, 3]), 'population': array([10, 20, 30]), } self._write_data_to_year(data, cache_dir, 2000) attribute_cache = AttributeCache(cache_directory=cache_dir) SimulationState(new_instance=True, base_cache_dir=self.temp_dir) SimulationState().set_cache_directory(cache_dir) SessionConfiguration(new_instance=True, in_storage=attribute_cache) SimulationState().set_current_time(2000) dataset_pool_2000 = DatasetPool(package_order=['urbansim'], storage=attribute_cache) dataset = dataset_pool_2000.get_dataset(self._dataset_name) variable_name = '%s.%s.percent_population_difference_from_2000' % ( self._package_name, self._dataset_name) dataset.compute_variables([variable_name], dataset_pool=dataset_pool_2000) pop_2000 = dataset.get_attribute(variable_name) self.assert_(ma.allequal(pop_2000, array([0, 0, 0])))
def prepare_for_simulation(self, config, cache_directory=None): self.config = Resources(config) base_cache_dir = self.config[ 'creating_baseyear_cache_configuration'].cache_directory_root self.simulation_state = SimulationState(new_instance=True, base_cache_dir=base_cache_dir, start_time=self.config.get( 'base_year', 0)) ### TODO: Get rid of this! There is no good reason to be changing the ### Configuration. if self.config['cache_directory'] is None: self.config[ 'cache_directory'] = self.simulation_state.get_cache_directory( ) SessionConfiguration( new_instance=True, package_order=self.config['dataset_pool_configuration']. package_order, in_storage=AttributeCache()) if config['creating_baseyear_cache_configuration'].cache_from_database: ForkProcess().fork_new_process( self.config['creating_baseyear_cache_configuration']. cache_scenario_database, self.config) else: CacheFltData().run(self.config)
def test(self): opus_core_path = OpusPackage().get_opus_core_path() dbf_directory = os.path.join(opus_core_path, 'tests', 'data', 'dbf') table_name = 'test_logical' cache_directory = self._temp_dir year = 1000 exporter = ExportDbfTableToCacheCommand( dbf_directory=dbf_directory, table_name=table_name, cache_directory=cache_directory, year=year, ) exporter.execute() attribute_cache = AttributeCache(cache_directory=cache_directory) old_time = SimulationState().get_current_time() SimulationState().set_current_time(year) values = attribute_cache.load_table(table_name) self.assertEqual(set(['keyid', 'works']), set(values.keys())) self.assert_(ma.allequal(array([1, 2, 3, 4, 5]), values['keyid'])) self.assert_(ma.allequal(array([1, 1, -1, 0, 0]), values['works'])) SimulationState().set_current_time(old_time)
def test(self): urbansim_constant = UrbansimConstantDataset( in_storage=AttributeCache()) self.assertEqual(urbansim_constant['absolute_min_year'], 1800) self.assertAlmostEqual(urbansim_constant['acres'], 150 * 150 * 0.0002471, 6) self.assert_(urbansim_constant["walking_distance_footprint"].ndim == 2)
def prepare_for_simulation(self, run_configuration, cache_directory=None): self.config = Resources(run_configuration) self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory) ### TODO: Get rid of this! There is no good reason to be changing the ### Configuration. if self.config['cache_directory'] is None: self.config[ 'cache_directory'] = self.simulation_state.get_cache_directory( ) SessionConfiguration( new_instance=True, package_order=self.config['dataset_pool_configuration']. package_order, in_storage=AttributeCache()) ForkProcess().fork_new_process( self.config['creating_baseyear_cache_configuration']. cache_scenario_database, self.config) # Create output database (normally done by run manager) if 'estimation_database_configuration' in self.config: db_server = DatabaseServer( self.config['estimation_database_configuration']) if not db_server.has_database( self.config['estimation_database_configuration']. database_name): db_server.create_database( self.config['estimation_database_configuration']. database_name)
def get_travel_data_from_travel_model(self, config, year, zone_set): """ """ logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...') input_directory = os.path.join(config['root'], 'opus_matsim', 'tmp') logger.log_status("input_directory: " + input_directory) in_storage = csv_storage(storage_location=input_directory) table_name = "travel_data" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) cache_storage = AttributeCache().get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) ##TODO:This may not work or may be wrong after the id_names of travel_data ##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang) existing_travel_data_set.join( travel_data_set, travel_data_set.get_non_id_primary_attribute_names(), metadata=AttributeType.PRIMARY) return existing_travel_data_set
def _set_cache_directory(self, cache_directory): if cache_directory != SimulationState().get_cache_directory(): SimulationState().set_cache_directory(cache_directory) SessionConfiguration( new_instance=True, package_order=self.dataset_pool_configuration.package_order, in_storage=AttributeCache())
def compare_travel_data_sets(self): # get copied travel data csv copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp') if not os.path.exists(copied_travel_data_location): raise StandardError('Travel data not found: %s' % copied_travel_data_location) logger.log_status('Get copied travel data: %s' % copied_travel_data_location) # convert travel data csv into travel data set matrix in_storage = csv_storage(storage_location = copied_travel_data_location) table_name = "travel_data" travel_data_attribute = 'single_vehicle_to_work_travel_cost' travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) # get exsisting travel data set and convert it also into travel data set matrix year = self.run_config['base_year']+2 attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory']) cache_storage = attribute_cache.get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name ) existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) from numpy import savetxt # for debugging savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f") savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f") # compare both data set matices compare = travel_data_attribute_mat == existing_travel_data_attribute_mat # return result return compare.all()
def _check_dataset_methods_on_dataset_view(self, ds, years_to_merge): self.assert_(ds is not None) ds.load_dataset(attributes='*', in_table_name='tests', in_storage=AttributeCache()) id = ds.get_attribute('id') attr1 = ds.get_attribute('attr1') # Does compute_variables work? ds.compute_variables(['opus_core.test.attr1_times_2']) attr1_times_2 = ds.get_attribute('attr1_times_2') # Are values as expected? self.assert_(ma.allequal(attr1 * 2, attr1_times_2)) # Does results have expected number of elements? self.assertEqual(len(years_to_merge) * 3, len(attr1_times_2)) # Does _compute_if_needed work? ds._compute_if_needed( 'opus_core.test.attr2_times_2', dataset_pool=SessionConfiguration().get_dataset_pool()) attr2_times_2 = ds.get_attribute('attr2_times_2') attr2 = ds.get_attribute('attr2') self.assert_(ma.allequal(attr2 * 2, attr2_times_2))
def __init__(self, config): ss = SimulationState(new_instance=True) ss.set_current_time(config['base_year']) ss.set_cache_directory(config['cache_directory']) SessionConfiguration(new_instance=True, package_order=config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) #if not os.path.exists(config['cache_directory']): ## if cache exists, it will automatically skip cacher = CreateBaseyearCache() cache_dir = cacher.run(config) if 'estimation_database_configuration' in config: db_server = DatabaseServer(config['estimation_database_configuration']) db = db_server.get_database(config['estimation_database_configuration'].database_name) out_storage = StorageFactory().get_storage( 'sql_storage', storage_location = db) else: output_cache = os.path.join(config['cache_directory'], str(config['base_year']+1)) out_storage = StorageFactory().get_storage('flt_storage', storage_location=output_cache) dataset_pool = SessionConfiguration().get_dataset_pool() households = dataset_pool.get_dataset("household") buildings = dataset_pool.get_dataset("building") zones = dataset_pool.get_dataset("zone") zone_ids = zones.get_id_attribute() capacity_attribute_name = "residential_units" #_of_use_id_%s" % id capacity_variable_name = "%s=sanfrancisco.zone.aggregate_%s_from_building" % \ (capacity_attribute_name, capacity_attribute_name) buildings.compute_variables("sanfrancisco.building.zone_id", dataset_pool=dataset_pool) zones.compute_variables(capacity_variable_name, dataset_pool=dataset_pool) building_zone_id = buildings.get_attribute('zone_id') # is_household_unplace = datasets['household'].get_attribute("building_id") <= 0 is_household_unplaced = 1 #all households are unplaced household_building_id = zeros(households.size(), dtype='int32')-1 #datasets['household'].get_attribute("building_id") for zone_id in zone_ids: capacity = zones.get_attribute_by_id(capacity_attribute_name, zone_id) is_household_in_this_zone = (households.get_attribute('zone_id') == zone_id) is_unplaced_household_in_this_zone = is_household_in_this_zone * is_household_unplaced is_building_in_this_zone = (building_zone_id == zone_id) # if not is_household_in_this_zone.sum() <= capacity: if capacity == 0 or is_household_in_this_zone.sum()==0: print "WARNING: zone %s has %s households but only %s units" % (zone_id, is_household_in_this_zone.sum(), capacity) continue prob = buildings.get_attribute(capacity_attribute_name) * is_building_in_this_zone / array(capacity, dtype=float64) r = random(sum(is_unplaced_household_in_this_zone)) prob_cumsum = ncumsum(prob) index_to_bldg = searchsorted(prob_cumsum, r) household_building_id[where(is_unplaced_household_in_this_zone)] = buildings.get_attribute_by_index('building_id', index_to_bldg) # import pdb;pdb.set_trace() households.set_values_of_one_attribute('building_id', household_building_id) households.write_dataset(out_table_name='households', out_storage=out_storage)
def run(self, location_set, development_event_set, *args, **kwargs): changed_indices, processed_development_event_indices = \ EventsCoordinator.run(self, location_set, development_event_set, *args, **kwargs) if development_event_set is not None: subset = DatasetSubset(development_event_set, processed_development_event_indices) subset.write_dataset(out_storage=AttributeCache()) return (changed_indices, processed_development_event_indices)
def flush_datasets_after_model(self, resources): if resources.get('flush_variables', False): AttributeCache().delete_computed_tables() # this will also delete computed attributes datasets_to_cache = SessionConfiguration().get_dataset_pool( ).datasets_in_pool().keys() else: datasets_to_cache = resources.get( "datasets_to_cache_after_each_model", []) self.flush_datasets(datasets_to_cache, after_model=True)
def test(self): # Set up a test cache storage = AttributeCache(cache_directory=self._temp_dir) SimulationState().set_current_time(2000) table_name = 'foo' values = { 'attribute1': array([1, 2, 3], dtype=int32), 'attribute2': array([4, 5, 6], dtype=int32), } storage.write_table(table_name, values) table_dir = os.path.join(self._temp_dir, '2000', table_name) self.assert_(os.path.exists(table_dir)) actual = set(os.listdir(table_dir)) expected = set([ 'attribute1.%(endian)si4' % replacements, 'attribute2.%(endian)si4' % replacements ]) self.assertEqual(expected, actual) exporter = ExportCacheToDbfTableCommand( cache_directory=self._temp_dir, year='2000', table_name=table_name, dbf_directory=self._temp_dir, decimalcount=4, ) exporter.execute() out_storage = dbf_storage(self._temp_dir) db = _dbf_class(out_storage._get_file_path_for_table(table_name)) length = max([len(values[key]) for key in values.keys()]) i = 0 field_type = {} for name, type in [ field.fieldInfo()[:2] for field in db.header.fields ]: field_type[name] = type for rec in db: for key in values.keys(): if field_type[key.upper()] is 'F': self.assertAlmostEqual(values[key][i], rec[key], 4) else: self.assertEqual(values[key][i], rec[key]) i = i + 1 self.assertEquals( length, i, msg="More values expected than the dbf file contains") db.close()
def __init__(self, package_name, attribute_name, lag_offset, dataset_name, index_name): self.package_name = package_name self.attribute_name = attribute_name self.simulation_state = SimulationState() self.attribute_cache = AttributeCache() self.lag_offset = lag_offset self.dataset_name = dataset_name self.index_name = index_name self.lag_index_name = "%s_lag%d" % (index_name, lag_offset) Variable.__init__(self)