class RunSimulationFromMysql: def prepare_for_simulation(self, run_configuration, cache_directory=None): self.config = Resources(run_configuration) self.simulation_state = SimulationState( new_instance=True, base_cache_dir=cache_directory, start_time=self.config.get("base_year", 0) ) ### TODO: Get rid of this! There is no good reason to be changing the ### Configuration. if self.config["cache_directory"] is None: self.config["cache_directory"] = self.simulation_state.get_cache_directory() SessionConfiguration( new_instance=True, package_order=self.config["dataset_pool_configuration"].package_order, in_storage=AttributeCache(), ) ForkProcess().fork_new_process( self.config["creating_baseyear_cache_configuration"].cache_scenario_database, self.config ) # Create output database (normally done by run manager) if "estimation_database_configuration" in self.config: db_server = DatabaseServer(self.config["estimation_database_configuration"]) if not db_server.has_database(self.config["estimation_database_configuration"].database_name): db_server.create_database(self.config["estimation_database_configuration"].database_name) def run_simulation(self, simulation_instance=None): logger.start_block("Simulation on database %s" % self.config["scenario_database_configuration"].database_name) try: if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) # simulation_instance.run_multiprocess(self.config, is_run_subset=True) finally: logger.end_block() logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory()) def cleanup(self, remove_cache, remove_output_database): """Remove all outputs of this simulation.""" self.simulation_state.remove_singleton(delete_cache=remove_cache) # Remove SessionConfiguration singleton, if it exists Singleton().remove_singleton_for_class(SessionConfiguration) cache_dir = self.config["cache_directory"] if os.path.exists(cache_dir): rmtree(cache_dir) if remove_output_database and ("estimation_database_configuration" in self.config): db_server = DatabaseServer(self.config["estimation_database_configuration"]) db_server.drop_database(self.config["estimation_database_configuration"].database_name) def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True): self.prepare_for_simulation(run_configuration) self.run_simulation(simulation_instance) self.cleanup(remove_cache)
class RunSimulation(object): def prepare_for_simulation(self, config, cache_directory=None): self.config = Resources(config) base_cache_dir = self.config[ 'creating_baseyear_cache_configuration'].cache_directory_root self.simulation_state = SimulationState(new_instance=True, base_cache_dir=base_cache_dir, start_time=self.config.get( 'base_year', 0)) ### TODO: Get rid of this! There is no good reason to be changing the ### Configuration. if self.config['cache_directory'] is None: self.config[ 'cache_directory'] = self.simulation_state.get_cache_directory( ) SessionConfiguration( new_instance=True, package_order=self.config['dataset_pool_configuration']. package_order, in_storage=AttributeCache()) if config['creating_baseyear_cache_configuration'].cache_from_database: ForkProcess().fork_new_process( self.config['creating_baseyear_cache_configuration']. cache_scenario_database, self.config) else: CacheFltData().run(self.config) def run_simulation(self, simulation_instance=None): if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) #simulation_instance.run_multiprocess(self.config, is_run_subset=True) logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory()) def cleanup(self, remove_cache=True): """Remove all outputs of this simulation.""" self.simulation_state.remove_singleton(delete_cache=remove_cache) SessionConfiguration().remove_singleton() if remove_cache: cache_dir = self.config['cache_directory'] if os.path.exists(cache_dir): rmtree(cache_dir) def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True): self.prepare_for_simulation(run_configuration) self.run_simulation(simulation_instance) self.cleanup(remove_cache)
class RunSimulationFromMysql: def prepare_for_simulation(self, run_configuration, cache_directory=None): self.config = Resources(run_configuration) self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory, start_time=self.config.get('base_year', 0)) ### TODO: Get rid of this! There is no good reason to be changing the ### Configuration. if self.config['cache_directory'] is None: self.config['cache_directory'] = self.simulation_state.get_cache_directory() SessionConfiguration(new_instance=True, package_order=self.config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) ForkProcess().fork_new_process(self.config['creating_baseyear_cache_configuration'].cache_scenario_database, self.config) # Create output database (normally done by run manager) if 'estimation_database_configuration' in self.config: db_server = DatabaseServer(self.config['estimation_database_configuration']) if not db_server.has_database(self.config['estimation_database_configuration'].database_name): db_server.create_database(self.config['estimation_database_configuration'].database_name) def run_simulation(self, simulation_instance=None): logger.start_block('Simulation on database %s' % self.config['scenario_database_configuration'].database_name) try: if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) #simulation_instance.run_multiprocess(self.config, is_run_subset=True) finally: logger.end_block() logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory()) def cleanup(self, remove_cache, remove_output_database): """Remove all outputs of this simulation.""" self.simulation_state.remove_singleton(delete_cache=remove_cache) # Remove SessionConfiguration singleton, if it exists Singleton().remove_singleton_for_class(SessionConfiguration) cache_dir = self.config['cache_directory'] if os.path.exists(cache_dir): rmtree(cache_dir) if remove_output_database and ('estimation_database_configuration' in self.config): db_server = DatabaseServer(self.config['estimation_database_configuration']) db_server.drop_database(self.config['estimation_database_configuration'].database_name) def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True): self.prepare_for_simulation(run_configuration) self.run_simulation(simulation_instance) self.cleanup(remove_cache)
class RunSimulation(object): def prepare_for_simulation(self, config, cache_directory=None): self.config = Resources(config) base_cache_dir = self.config['creating_baseyear_cache_configuration'].cache_directory_root self.simulation_state = SimulationState(new_instance=True, base_cache_dir=base_cache_dir, start_time=self.config.get('base_year', 0)) ### TODO: Get rid of this! There is no good reason to be changing the ### Configuration. if self.config['cache_directory'] is None: self.config['cache_directory'] = self.simulation_state.get_cache_directory() SessionConfiguration(new_instance=True, package_order=self.config['dataset_pool_configuration'].package_order, in_storage=AttributeCache()) if config['creating_baseyear_cache_configuration'].cache_from_database: ForkProcess().fork_new_process(self.config['creating_baseyear_cache_configuration'].cache_scenario_database, self.config) else: CacheFltData().run(self.config) def run_simulation(self, simulation_instance=None): if simulation_instance is None: simulation_instance = ModelSystem() simulation_instance.run(self.config) #simulation_instance.run_multiprocess(self.config, is_run_subset=True) logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory()) def cleanup(self, remove_cache=True): """Remove all outputs of this simulation.""" self.simulation_state.remove_singleton(delete_cache=remove_cache) SessionConfiguration().remove_singleton() if remove_cache: cache_dir = self.config['cache_directory'] if os.path.exists(cache_dir): rmtree(cache_dir) def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True): self.prepare_for_simulation(run_configuration) self.run_simulation(simulation_instance) self.cleanup(remove_cache)
class TestLagVariables(opus_unittest.OpusTestCase): def setUp(self): self.config = TestCacheConfiguration() self.simulation_state = SimulationState(new_instance=True) SessionConfiguration(self.config, new_instance=True, package_order=['urbansim', 'opus_core'], in_storage=AttributeCache()) self.base_year = self.config['base_year'] creating_baseyear_cache_configuration = self.config['creating_baseyear_cache_configuration'] self.simulation_state.set_current_time(self.base_year) cache_directory = self.simulation_state.get_cache_directory() copytree(os.path.join(creating_baseyear_cache_configuration.baseyear_cache.existing_cache_to_copy, str(self.base_year)), os.path.join(cache_directory, str(self.base_year))) cacher = CacheScenarioDatabase() cacher.prepare_data_before_baseyear(cache_directory, self.base_year, creating_baseyear_cache_configuration) self.config['cache_directory'] = cache_directory cache_storage = AttributeCache().get_flt_storage_for_year(self.base_year) cache_directory = self.simulation_state.get_cache_directory() flt_directory = os.path.join(cache_directory, str(self.base_year)) self.gridcell = DatasetFactory().get_dataset('gridcell', package='urbansim', subdir='datasets', arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)} ) def tearDown(self): self.simulation_state.remove_singleton(delete_cache=True) def test_lag_variables(self): """Test lag variables""" # A weak test that computing a lag variable on a realistic dataset does not crash. self.gridcell.compute_variables('urbansim.gridcell.n_recent_transitions_to_developed', resources=self.config) # The following tests are fragile, since they need to know exactly what values are being # subtracted, and ignore any negative amount that is truncated at zero. # If you change the "subset" dataset to a different region, you will # have to update the expected value. self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft', resources=self.config) self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft_lag1', resources=self.config) self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft_lag2', resources=self.config) sqft = self.gridcell.get_attribute('commercial_sqft').sum() sqft_lag1 = self.gridcell.get_attribute('commercial_sqft_lag1').sum() sqft_lag2 = self.gridcell.get_attribute('commercial_sqft_lag2').sum() logger.log_status('sqft = %s' % sqft) logger.log_status('sqft_lag1 = %s' % sqft_lag1) logger.log_status('sqft_lag2 = %s' % sqft_lag2) logger.log_status('base_year = %s' % self.base_year) self.assertEqual(self.base_year, SimulationState().get_current_time()) self.assertEqual(sqft, sqft_lag1) self.assertEqual(578+2083+1103+87, sqft_lag1 - sqft_lag2) # Do lag variables produce different results for derived attributes? self.gridcell.compute_variables('urbansim.gridcell.n_recent_development_projects', resources=self.config) self.gridcell.compute_variables('urbansim.gridcell.n_recent_development_projects_lag1', resources=self.config) n_recent_projects = self.gridcell.get_attribute('n_recent_development_projects').sum() n_recent_projects_lag1 = self.gridcell.get_attribute('n_recent_development_projects_lag1').sum() self.assertEqual(n_recent_projects, 11) self.assertEqual(n_recent_projects_lag1, 15) # Do lag_variables produce different results for derived attributes without lags? self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft', resources=self.config) self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft_lag4', resources=self.config) sqft = self.gridcell.get_attribute('ln_commercial_sqft').sum() sqft_lag4 = self.gridcell.get_attribute('ln_commercial_sqft_lag4').sum() self.assertNotEqual(sqft, sqft_lag4)
class TestLagVariables(opus_unittest.OpusTestCase): def setUp(self): self.config = TestCacheConfiguration() self.simulation_state = SimulationState(new_instance=True) SessionConfiguration(self.config, new_instance=True, package_order=['urbansim', 'opus_core'], in_storage=AttributeCache()) self.base_year = self.config['base_year'] creating_baseyear_cache_configuration = self.config[ 'creating_baseyear_cache_configuration'] self.simulation_state.set_current_time(self.base_year) cache_directory = self.simulation_state.get_cache_directory() copytree( os.path.join( creating_baseyear_cache_configuration.baseyear_cache. existing_cache_to_copy, str(self.base_year)), os.path.join(cache_directory, str(self.base_year))) cacher = CacheScenarioDatabase() cacher.prepare_data_before_baseyear( cache_directory, self.base_year, creating_baseyear_cache_configuration) self.config['cache_directory'] = cache_directory cache_storage = AttributeCache().get_flt_storage_for_year( self.base_year) cache_directory = self.simulation_state.get_cache_directory() flt_directory = os.path.join(cache_directory, str(self.base_year)) self.gridcell = DatasetFactory().get_dataset( 'gridcell', package='urbansim', subdir='datasets', arguments={ 'in_storage': StorageFactory().get_storage('flt_storage', storage_location=flt_directory) }) def tearDown(self): self.simulation_state.remove_singleton(delete_cache=True) def test_lag_variables(self): """Test lag variables""" # A weak test that computing a lag variable on a realistic dataset does not crash. self.gridcell.compute_variables( 'urbansim.gridcell.n_recent_transitions_to_developed', resources=self.config) # The following tests are fragile, since they need to know exactly what values are being # subtracted, and ignore any negative amount that is truncated at zero. # If you change the "subset" dataset to a different region, you will # have to update the expected value. self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft', resources=self.config) self.gridcell.compute_variables( 'urbansim.gridcell.commercial_sqft_lag1', resources=self.config) self.gridcell.compute_variables( 'urbansim.gridcell.commercial_sqft_lag2', resources=self.config) sqft = self.gridcell.get_attribute('commercial_sqft').sum() sqft_lag1 = self.gridcell.get_attribute('commercial_sqft_lag1').sum() sqft_lag2 = self.gridcell.get_attribute('commercial_sqft_lag2').sum() logger.log_status('sqft = %s' % sqft) logger.log_status('sqft_lag1 = %s' % sqft_lag1) logger.log_status('sqft_lag2 = %s' % sqft_lag2) logger.log_status('base_year = %s' % self.base_year) self.assertEqual(self.base_year, SimulationState().get_current_time()) self.assertEqual(sqft, sqft_lag1) self.assertEqual(578 + 2083 + 1103 + 87, sqft_lag1 - sqft_lag2) # Do lag variables produce different results for derived attributes? self.gridcell.compute_variables( 'urbansim.gridcell.n_recent_development_projects', resources=self.config) self.gridcell.compute_variables( 'urbansim.gridcell.n_recent_development_projects_lag1', resources=self.config) n_recent_projects = self.gridcell.get_attribute( 'n_recent_development_projects').sum() n_recent_projects_lag1 = self.gridcell.get_attribute( 'n_recent_development_projects_lag1').sum() self.assertEqual(n_recent_projects, 11) self.assertEqual(n_recent_projects_lag1, 15) # Do lag_variables produce different results for derived attributes without lags? self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft', resources=self.config) self.gridcell.compute_variables( 'urbansim.gridcell.ln_commercial_sqft_lag4', resources=self.config) sqft = self.gridcell.get_attribute('ln_commercial_sqft').sum() sqft_lag4 = self.gridcell.get_attribute( 'ln_commercial_sqft_lag4').sum() self.assertNotEqual(sqft, sqft_lag4)