class RunSimulationFromMysql:
def prepare_for_simulation(self, run_configuration, cache_directory=None):
self.config = Resources(run_configuration)
self.simulation_state = SimulationState(
new_instance=True, base_cache_dir=cache_directory, start_time=self.config.get("base_year", 0)
)
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config["cache_directory"] is None:
self.config["cache_directory"] = self.simulation_state.get_cache_directory()
SessionConfiguration(
new_instance=True,
package_order=self.config["dataset_pool_configuration"].package_order,
in_storage=AttributeCache(),
)
ForkProcess().fork_new_process(
self.config["creating_baseyear_cache_configuration"].cache_scenario_database, self.config
)
# Create output database (normally done by run manager)
if "estimation_database_configuration" in self.config:
db_server = DatabaseServer(self.config["estimation_database_configuration"])
if not db_server.has_database(self.config["estimation_database_configuration"].database_name):
db_server.create_database(self.config["estimation_database_configuration"].database_name)
def run_simulation(self, simulation_instance=None):
logger.start_block("Simulation on database %s" % self.config["scenario_database_configuration"].database_name)
try:
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
# simulation_instance.run_multiprocess(self.config, is_run_subset=True)
finally:
logger.end_block()
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def cleanup(self, remove_cache, remove_output_database):
"""Remove all outputs of this simulation."""
self.simulation_state.remove_singleton(delete_cache=remove_cache)
# Remove SessionConfiguration singleton, if it exists
Singleton().remove_singleton_for_class(SessionConfiguration)
cache_dir = self.config["cache_directory"]
if os.path.exists(cache_dir):
rmtree(cache_dir)
if remove_output_database and ("estimation_database_configuration" in self.config):
db_server = DatabaseServer(self.config["estimation_database_configuration"])
db_server.drop_database(self.config["estimation_database_configuration"].database_name)
def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True):
self.prepare_for_simulation(run_configuration)
self.run_simulation(simulation_instance)
self.cleanup(remove_cache)
class RunSimulation(object):
def prepare_for_simulation(self, config, cache_directory=None):
self.config = Resources(config)
base_cache_dir = self.config[
'creating_baseyear_cache_configuration'].cache_directory_root
self.simulation_state = SimulationState(new_instance=True,
base_cache_dir=base_cache_dir,
start_time=self.config.get(
'base_year', 0))
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config[
'cache_directory'] = self.simulation_state.get_cache_directory(
)
SessionConfiguration(
new_instance=True,
package_order=self.config['dataset_pool_configuration'].
package_order,
in_storage=AttributeCache())
if config['creating_baseyear_cache_configuration'].cache_from_database:
ForkProcess().fork_new_process(
self.config['creating_baseyear_cache_configuration'].
cache_scenario_database, self.config)
else:
CacheFltData().run(self.config)
def run_simulation(self, simulation_instance=None):
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
logger.log_status("Data cache in %s" %
self.simulation_state.get_cache_directory())
def cleanup(self, remove_cache=True):
"""Remove all outputs of this simulation."""
self.simulation_state.remove_singleton(delete_cache=remove_cache)
SessionConfiguration().remove_singleton()
if remove_cache:
cache_dir = self.config['cache_directory']
if os.path.exists(cache_dir):
rmtree(cache_dir)
def prepare_and_run(self,
run_configuration,
simulation_instance=None,
remove_cache=True):
self.prepare_for_simulation(run_configuration)
self.run_simulation(simulation_instance)
self.cleanup(remove_cache)
class RunSimulationFromMysql:
def prepare_for_simulation(self, run_configuration, cache_directory=None):
self.config = Resources(run_configuration)
self.simulation_state = SimulationState(new_instance=True, base_cache_dir=cache_directory,
start_time=self.config.get('base_year', 0))
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config['cache_directory'] = self.simulation_state.get_cache_directory()
SessionConfiguration(new_instance=True,
package_order=self.config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
ForkProcess().fork_new_process(self.config['creating_baseyear_cache_configuration'].cache_scenario_database, self.config)
# Create output database (normally done by run manager)
if 'estimation_database_configuration' in self.config:
db_server = DatabaseServer(self.config['estimation_database_configuration'])
if not db_server.has_database(self.config['estimation_database_configuration'].database_name):
db_server.create_database(self.config['estimation_database_configuration'].database_name)
def run_simulation(self, simulation_instance=None):
logger.start_block('Simulation on database %s'
% self.config['scenario_database_configuration'].database_name)
try:
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
finally:
logger.end_block()
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def cleanup(self, remove_cache, remove_output_database):
"""Remove all outputs of this simulation."""
self.simulation_state.remove_singleton(delete_cache=remove_cache)
# Remove SessionConfiguration singleton, if it exists
Singleton().remove_singleton_for_class(SessionConfiguration)
cache_dir = self.config['cache_directory']
if os.path.exists(cache_dir):
rmtree(cache_dir)
if remove_output_database and ('estimation_database_configuration' in self.config):
db_server = DatabaseServer(self.config['estimation_database_configuration'])
db_server.drop_database(self.config['estimation_database_configuration'].database_name)
def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True):
self.prepare_for_simulation(run_configuration)
self.run_simulation(simulation_instance)
self.cleanup(remove_cache)
class RunSimulation(object):
def prepare_for_simulation(self, config, cache_directory=None):
self.config = Resources(config)
base_cache_dir = self.config['creating_baseyear_cache_configuration'].cache_directory_root
self.simulation_state = SimulationState(new_instance=True, base_cache_dir=base_cache_dir,
start_time=self.config.get('base_year', 0))
### TODO: Get rid of this! There is no good reason to be changing the
### Configuration.
if self.config['cache_directory'] is None:
self.config['cache_directory'] = self.simulation_state.get_cache_directory()
SessionConfiguration(new_instance=True,
package_order=self.config['dataset_pool_configuration'].package_order,
in_storage=AttributeCache())
if config['creating_baseyear_cache_configuration'].cache_from_database:
ForkProcess().fork_new_process(self.config['creating_baseyear_cache_configuration'].cache_scenario_database, self.config)
else:
CacheFltData().run(self.config)
def run_simulation(self, simulation_instance=None):
if simulation_instance is None:
simulation_instance = ModelSystem()
simulation_instance.run(self.config)
#simulation_instance.run_multiprocess(self.config, is_run_subset=True)
logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
def cleanup(self, remove_cache=True):
"""Remove all outputs of this simulation."""
self.simulation_state.remove_singleton(delete_cache=remove_cache)
SessionConfiguration().remove_singleton()
if remove_cache:
cache_dir = self.config['cache_directory']
if os.path.exists(cache_dir):
rmtree(cache_dir)
def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True):
self.prepare_for_simulation(run_configuration)
self.run_simulation(simulation_instance)
self.cleanup(remove_cache)
class TestLagVariables(opus_unittest.OpusTestCase):
def setUp(self):
self.config = TestCacheConfiguration()
self.simulation_state = SimulationState(new_instance=True)
SessionConfiguration(self.config, new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = self.config['base_year']
creating_baseyear_cache_configuration = self.config['creating_baseyear_cache_configuration']
self.simulation_state.set_current_time(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
copytree(os.path.join(creating_baseyear_cache_configuration.baseyear_cache.existing_cache_to_copy,
str(self.base_year)),
os.path.join(cache_directory, str(self.base_year)))
cacher = CacheScenarioDatabase()
cacher.prepare_data_before_baseyear(cache_directory, self.base_year, creating_baseyear_cache_configuration)
self.config['cache_directory'] = cache_directory
cache_storage = AttributeCache().get_flt_storage_for_year(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
self.gridcell = DatasetFactory().get_dataset('gridcell',
package='urbansim',
subdir='datasets',
arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)}
)
def tearDown(self):
self.simulation_state.remove_singleton(delete_cache=True)
def test_lag_variables(self):
"""Test lag variables"""
# A weak test that computing a lag variable on a realistic dataset does not crash.
self.gridcell.compute_variables('urbansim.gridcell.n_recent_transitions_to_developed',
resources=self.config)
# The following tests are fragile, since they need to know exactly what values are being
# subtracted, and ignore any negative amount that is truncated at zero.
# If you change the "subset" dataset to a different region, you will
# have to update the expected value.
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft_lag1',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft_lag2',
resources=self.config)
sqft = self.gridcell.get_attribute('commercial_sqft').sum()
sqft_lag1 = self.gridcell.get_attribute('commercial_sqft_lag1').sum()
sqft_lag2 = self.gridcell.get_attribute('commercial_sqft_lag2').sum()
logger.log_status('sqft = %s' % sqft)
logger.log_status('sqft_lag1 = %s' % sqft_lag1)
logger.log_status('sqft_lag2 = %s' % sqft_lag2)
logger.log_status('base_year = %s' % self.base_year)
self.assertEqual(self.base_year, SimulationState().get_current_time())
self.assertEqual(sqft, sqft_lag1)
self.assertEqual(578+2083+1103+87, sqft_lag1 - sqft_lag2)
# Do lag variables produce different results for derived attributes?
self.gridcell.compute_variables('urbansim.gridcell.n_recent_development_projects',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.n_recent_development_projects_lag1',
resources=self.config)
n_recent_projects = self.gridcell.get_attribute('n_recent_development_projects').sum()
n_recent_projects_lag1 = self.gridcell.get_attribute('n_recent_development_projects_lag1').sum()
self.assertEqual(n_recent_projects, 11)
self.assertEqual(n_recent_projects_lag1, 15)
# Do lag_variables produce different results for derived attributes without lags?
self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft',
resources=self.config)
self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft_lag4',
resources=self.config)
sqft = self.gridcell.get_attribute('ln_commercial_sqft').sum()
sqft_lag4 = self.gridcell.get_attribute('ln_commercial_sqft_lag4').sum()
self.assertNotEqual(sqft, sqft_lag4)
class TestLagVariables(opus_unittest.OpusTestCase):
def setUp(self):
self.config = TestCacheConfiguration()
self.simulation_state = SimulationState(new_instance=True)
SessionConfiguration(self.config,
new_instance=True,
package_order=['urbansim', 'opus_core'],
in_storage=AttributeCache())
self.base_year = self.config['base_year']
creating_baseyear_cache_configuration = self.config[
'creating_baseyear_cache_configuration']
self.simulation_state.set_current_time(self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
copytree(
os.path.join(
creating_baseyear_cache_configuration.baseyear_cache.
existing_cache_to_copy, str(self.base_year)),
os.path.join(cache_directory, str(self.base_year)))
cacher = CacheScenarioDatabase()
cacher.prepare_data_before_baseyear(
cache_directory, self.base_year,
creating_baseyear_cache_configuration)
self.config['cache_directory'] = cache_directory
cache_storage = AttributeCache().get_flt_storage_for_year(
self.base_year)
cache_directory = self.simulation_state.get_cache_directory()
flt_directory = os.path.join(cache_directory, str(self.base_year))
self.gridcell = DatasetFactory().get_dataset(
'gridcell',
package='urbansim',
subdir='datasets',
arguments={
'in_storage':
StorageFactory().get_storage('flt_storage',
storage_location=flt_directory)
})
def tearDown(self):
self.simulation_state.remove_singleton(delete_cache=True)
def test_lag_variables(self):
"""Test lag variables"""
# A weak test that computing a lag variable on a realistic dataset does not crash.
self.gridcell.compute_variables(
'urbansim.gridcell.n_recent_transitions_to_developed',
resources=self.config)
# The following tests are fragile, since they need to know exactly what values are being
# subtracted, and ignore any negative amount that is truncated at zero.
# If you change the "subset" dataset to a different region, you will
# have to update the expected value.
self.gridcell.compute_variables('urbansim.gridcell.commercial_sqft',
resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.commercial_sqft_lag1', resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.commercial_sqft_lag2', resources=self.config)
sqft = self.gridcell.get_attribute('commercial_sqft').sum()
sqft_lag1 = self.gridcell.get_attribute('commercial_sqft_lag1').sum()
sqft_lag2 = self.gridcell.get_attribute('commercial_sqft_lag2').sum()
logger.log_status('sqft = %s' % sqft)
logger.log_status('sqft_lag1 = %s' % sqft_lag1)
logger.log_status('sqft_lag2 = %s' % sqft_lag2)
logger.log_status('base_year = %s' % self.base_year)
self.assertEqual(self.base_year, SimulationState().get_current_time())
self.assertEqual(sqft, sqft_lag1)
self.assertEqual(578 + 2083 + 1103 + 87, sqft_lag1 - sqft_lag2)
# Do lag variables produce different results for derived attributes?
self.gridcell.compute_variables(
'urbansim.gridcell.n_recent_development_projects',
resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.n_recent_development_projects_lag1',
resources=self.config)
n_recent_projects = self.gridcell.get_attribute(
'n_recent_development_projects').sum()
n_recent_projects_lag1 = self.gridcell.get_attribute(
'n_recent_development_projects_lag1').sum()
self.assertEqual(n_recent_projects, 11)
self.assertEqual(n_recent_projects_lag1, 15)
# Do lag_variables produce different results for derived attributes without lags?
self.gridcell.compute_variables('urbansim.gridcell.ln_commercial_sqft',
resources=self.config)
self.gridcell.compute_variables(
'urbansim.gridcell.ln_commercial_sqft_lag4', resources=self.config)
sqft = self.gridcell.get_attribute('ln_commercial_sqft').sum()
sqft_lag4 = self.gridcell.get_attribute(
'ln_commercial_sqft_lag4').sum()
self.assertNotEqual(sqft, sqft_lag4)
