def prepare_specification_and_coefficients(specification_storage=None,
specification_table=None,
coefficients_storage=None,
coefficients_table=None,
sample_coefficients=False,
cache_storage=None,
**kwargs):
""" Load specification and coefficients from given tables in given storages.
If 'sample_coefficients' is True, coefficients are sampled from given distribution. In such a case,
either an argument 'distribution' should be given (equal either 'normal' or 'uniform'),
or argument distribution_dictionary should be given containing details about sampling specific coefficients
(see docstring for method sample_values in opus_core/coefficients.py).
If coefficients are sampled, the new values are flushed into cache.
"""
specification = None
if specification_storage is not None and specification_table is not None:
specification = EquationSpecification(in_storage=specification_storage)
specification.load(in_table_name=specification_table)
coefficients = None
if (coefficients_storage is not None) and (coefficients_table is not None):
coefficients = Coefficients(in_storage=coefficients_storage)
coefficients.load(in_table_name=coefficients_table)
if sample_coefficients:
coefficients = coefficients.sample_values(**kwargs)
coefficients.flush_coefficients(coefficients_table, cache_storage)
return (specification, coefficients)
def prepare_specification_and_coefficients(specification_storage=None,
specification_table=None,
coefficients_storage=None,
coefficients_table=None,
sample_coefficients=False,
cache_storage=None,
**kwargs):
""" Load specification and coefficients from given tables in given storages.
If 'sample_coefficients' is True, coefficients are sampled from given distribution. In such a case,
either an argument 'distribution' should be given (equal either 'normal' or 'uniform'),
or argument distribution_dictionary should be given containing details about sampling specific coefficients
(see docstring for method sample_values in opus_core/coefficients.py).
If coefficients are sampled, the new values are flushed into cache.
"""
specification = None
if specification_storage is not None and specification_table is not None:
specification = EquationSpecification(in_storage=specification_storage)
specification.load(in_table_name=specification_table)
coefficients = None
if (coefficients_storage is not None) and (coefficients_table is not None):
coefficients = Coefficients(in_storage=coefficients_storage)
coefficients.load(in_table_name=coefficients_table)
if sample_coefficients:
coefficients = coefficients.sample_values(**kwargs)
coefficients.flush_coefficients(coefficients_table, cache_storage)
return (specification, coefficients)
def run(self, base_directory, urbansim_cache_directory, years):
""" run the simulation
base_directory: directory contains all years folder of lccm.
urbansim_cache_directory: directory contains all years folder of urbansim cache.
years: lists of year to run."""
model = LandCoverChangeModel(self.possible_lcts,
submodel_string=self.lct_attribute,
choice_attribute_name=self.lct_attribute,
debuglevel=4)
coefficients = Coefficients()
storage = StorageFactory().get_storage('tab_storage',
storage_location=os.path.join(
self.package_path, 'data'))
coefficients.load(in_storage=storage,
in_table_name="land_cover_change_model_coefficients")
specification = EquationSpecification(in_storage=storage)
specification.load(
in_table_name="land_cover_change_model_specification")
specification.set_variable_prefix("biocomplexity.land_cover.")
constants = Constants()
simulation_state = SimulationState()
simulation_state.set_cache_directory(urbansim_cache_directory)
attribute_cache = AttributeCache()
index = arange(100000)
for year in years:
simulation_state.set_current_time(year)
#land_cover_path = os.path.join(base_directory, str(year))
land_cover_path = base_directory
land_covers = LandCoverDataset(
in_storage=StorageFactory().get_storage(
'flt_storage', storage_location=land_cover_path),
out_storage=StorageFactory().get_storage(
'flt_storage', storage_location=land_cover_path),
debuglevel=4)
land_covers.subset_by_index(index)
#land_covers.load_dataset()
gridcells = GridcellDataset(in_storage=attribute_cache,
debuglevel=4)
agents_index = None
model.run(specification,
coefficients,
land_covers,
data_objects={
"gridcell": gridcells,
"constants": constants,
"flush_variables": True
},
chunk_specification={'nchunks': 1})
land_covers.flush_dataset()
del gridcells
del land_covers
def run(self, base_directory, urbansim_cache_directory, years):
""" run the simulation
base_directory: directory contains all years folder of lccm.
urbansim_cache_directory: directory contains all years folder of urbansim cache.
years: lists of year to run."""
model = LandCoverChangeModel(self.possible_lcts, submodel_string=self.lct_attribute,
choice_attribute_name= self.lct_attribute, debuglevel=4)
coefficients = Coefficients()
storage = StorageFactory().get_storage('tab_storage',
storage_location=os.path.join(self.package_path, 'data'))
coefficients.load(in_storage=storage, in_table_name="land_cover_change_model_coefficients")
specification = EquationSpecification(in_storage=storage)
specification.load(in_table_name="land_cover_change_model_specification")
specification.set_variable_prefix("biocomplexity.land_cover.")
constants = Constants()
simulation_state = SimulationState()
simulation_state.set_cache_directory(urbansim_cache_directory)
attribute_cache = AttributeCache()
index = arange(100000)
for year in years:
simulation_state.set_current_time(year)
#land_cover_path = os.path.join(base_directory, str(year))
land_cover_path = base_directory
land_covers = LandCoverDataset(in_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
out_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
debuglevel=4)
land_covers.subset_by_index(index)
#land_covers.load_dataset()
gridcells = GridcellDataset(in_storage=attribute_cache, debuglevel=4)
agents_index = None
model.run(specification, coefficients, land_covers, data_objects={"gridcell":gridcells,
"constants":constants, "flush_variables":True},
chunk_specification = {'nchunks':1}
)
land_covers.flush_dataset()
del gridcells
del land_covers
from opus_core.variables.attribute_type import AttributeType
submodel_string_short_name = VariableName(submodel_string).get_short_name()
if submodel_string_short_name in agent_set.get_known_attribute_names():
agent_set.add_attribute(agent_set.get_attribute(submodel_string_short_name), "submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients, specification, neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants()
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc','urbansim','opus_core'],
storage=AttributeCache())
choice_set = dataset_pool.get_dataset(choice_set_name)
if choice_filter_index is None:
choice_filter_index = arange(choice_set.size())
def run(self, base_directory, urbansim_cache_directory, years, output_directory, temp_folder,
coefficients_name, specification_name, convert_flt=True, convert_input=False):
""" run the simulation
base_directory: directory contains all years folder of lccm.
urbansim_cache_directory: directory contains all years folder of urbansim cache.
years: lists of year to run."""
model = LandCoverChangeModel(self.possible_lcts, submodel_string=self.lct_attribute,
choice_attribute_name=self.lct_attribute, debuglevel=4)
coefficients = Coefficients()
storage = StorageFactory().get_storage('tab_storage',
storage_location=os.path.join(self.package_path, 'data'))
coefficients.load(in_storage=storage, in_table_name=coefficients_name)
specification = EquationSpecification(in_storage=storage)
specification.load(in_table_name=specification_name)
specification.set_variable_prefix("biocomplexity.land_cover.")
constants = Constants()
simulation_state = SimulationState()
simulation_state.set_cache_directory(urbansim_cache_directory)
attribute_cache = AttributeCache()
SessionConfiguration(new_instance=True,
package_order=['biocomplexity', 'urbansim', 'opus_core'],
in_storage=AttributeCache())
ncols = LccmConfiguration.ncols
if temp_folder is None:
self.temp_land_cover_dir = tempfile.mkdtemp()
else:
self.temp_land_cover_dir = temp_folder
for year in years:
land_cover_path = self._generate_input_land_cover(year, base_directory, urbansim_cache_directory,
years, output_directory, convert_flt, convert_input)
#max_size = 174338406 (orig) - act. int: 19019944 (37632028 incl NoData)
max_size = self._get_max_index(land_cover_path) # 1st instance of lc_dataset - but looks like a 'lite' version
offset = min(LccmConfiguration.offset, max_size)
s = 0
t = offset
while (s < t and t <= max_size):
logger.log_status("Offset: ", s, t)
index = arange(s,t)
land_cover_cache_path=os.path.join(urbansim_cache_directory,str(year),'land_covers')
self._clean_up_land_cover_cache(land_cover_cache_path)
simulation_state.set_current_time(year)
# 2nd instance of lc_dataset
land_covers = LandCoverDataset(in_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
out_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
debuglevel=4)
land_covers.subset_by_index(index)
# land_covers.load_dataset()
gridcells = GridcellDataset(in_storage=attribute_cache, debuglevel=4)
agents_index = None
model.run(specification, coefficients, land_covers, data_objects={"gridcell":gridcells,
"constants":constants, "flush_variables":True},
chunk_specification = {'nchunks':5}) ## chunk size set here
land_covers.flush_dataset()
del gridcells
del land_covers
# self._generate_output_flt(year, urbansim_cache_directory, output_directory, convert_flt)
self._generate_output_flt2(year, urbansim_cache_directory, output_directory, convert_flt)
if t >= max_size: break
s = max(t-10*ncols,s)
t = min(t+offset-10*ncols,max_size)
# clean up temp storage after done simulation
shutil.rmtree(self.temp_land_cover_dir)
# households from PSRC
agents_psrc = HouseholdDataset(in_storage=StorageFactory().get_storage(
'flt_storage', storage_location="/home/hana/bandera/urbansim/data/GPSRC"),
in_table_name="hh")
agents_psrc.summary()
dbcon = []
config = ScenarioDatabaseConfiguration()
server = DatabaseServer(config)
db = server.get_database('PSRC_2000_baseyear')
storage = StorageFactory().get_storage('sql_storage', storage_location=db)
coefficients = Coefficients(in_storage=storage)
coefficients.load(in_table_name="household_location_choice_model_coefficients")
specification = EquationSpecification(in_storage=storage)
specification.load(
in_table_name="household_location_choice_model_specification")
specification.get_variable_names()
hlcm_psrc = HouseholdLocationChoiceModelCreator().get_model(
location_set=locations_psrc,
sampler="opus_core.samplers.weighted_sampler",
sample_size_locations=10,
choices="urbansim.lottery_choices",
compute_capacity_flag=True,
run_config=Resources(
{"capacity_string": "urbansim.gridcell.vacant_residential_units"}))
result = hlcm_psrc.run(specification,
models = ['real_estate_price_model_coefficients', 'household_location_choice_model_coefficients', 'non_home_based_employment_location_choice_model_coefficients', 'home_based_employment_location_choice_model_coefficients', 'work_at_home_choice_model_coefficients', 'workplace_choice_model_for_resident_coefficients' ]
content_coefficients = '' # contains whole content for coefficients
content_specification = '' # contains whole content for specifications
# go trough all models ...
for m in range(len(models)):
model = models[m]
content_coefficients+='\n\r'
content_coefficients+='Model: %s\n\r' %model
# ... get their coefficients ...
coefficients = Coefficients(in_storage=storage)
coefficients.load(resources=None, in_storage=storage, in_table_name=model)
# ... prepare for printing ...
names = coefficients.names
estimates = coefficients.values
std_errors = coefficients.standard_errors
sub_ids = coefficients.submodels
t_stats = coefficients.other_measures['t_statistic']
# ... finally print out all available data
content_coefficients+='{0:40s} {1:10s} {2:10s} {3:10s} {4:10s}\n\r'.format('coefficient_name', 'estimate', 'error', 'submodel_id', 't_statistic')
for i in range(len(names)):
if len(sub_ids) <= 0:
#content_coefficients+='{0:40s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])#'{0:30s} {1:10f} {2:10f} {3:10s} {4:10f}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])
content_coefficients+='{0:40s} {1:10s} {2:10s} {3:10s} {4:10s}\n'.format(names[i], estimates[i], std_errors[i], '-', t_stats[i])
else:
if submodel_string_short_name in agent_set.get_known_attribute_names():
agent_set.add_attribute(
agent_set.get_attribute(submodel_string_short_name),
"submodel",
metadata=AttributeType.PRIMARY)
else:
agent_set.compute_variables("submodel = %s" % submodel_string)
specification_table = test_settings[model]['specification_table']
coefficients_table = test_settings[model]['coefficient_table']
base_cache_storage = AttributeCache().get_flt_storage_for_year(base_year)
specification = EquationSpecification(in_storage=base_cache_storage)
specification.load(in_table_name=specification_table)
coefficients = Coefficients(in_storage=base_cache_storage)
coefficients.load(in_table_name=coefficients_table)
specified_coefficients = SpecifiedCoefficients().create(coefficients,
specification,
neqs=1)
variables = specified_coefficients.get_full_variable_names_without_constants(
)
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc', 'urbansim', 'opus_core'],
storage=AttributeCache())
storage_location = "/home/hana/bandera/urbansim/data/GPSRC"),
in_table_name = "hh")
agents_psrc.summary()
dbcon = []
config = ScenarioDatabaseConfiguration()
server = DatabaseServer(config)
db = server.get_database('PSRC_2000_baseyear')
storage = StorageFactory().get_storage(
'sql_storage',
storage_location = db)
coefficients = Coefficients(in_storage=storage)
coefficients.load(in_table_name="household_location_choice_model_coefficients")
specification = EquationSpecification(in_storage=storage)
specification.load(in_table_name="household_location_choice_model_specification")
specification.get_variable_names()
hlcm_psrc = HouseholdLocationChoiceModelCreator().get_model(
location_set = locations_psrc,
sampler = "opus_core.samplers.weighted_sampler",
sample_size_locations=10,
choices="urbansim.lottery_choices",
compute_capacity_flag=True,
run_config=Resources({"capacity_string":"urbansim.gridcell.vacant_residential_units"}))
result = hlcm_psrc.run(specification, coefficients, agents_psrc,
agents_index=sample(range(agents_psrc.size()), 500),