def estimate(self, spec_var=None, spec_py=None,
submodel_string = "workers",
agent_sample_rate=0.005, alt_sample_size=None):
"""
"""
CLOSE = 0.001
sampler = "opus_core.samplers.weighted_sampler"
if alt_sample_size==None:
sampler = None
date_time_str=strftime("%Y_%m_%d__%H_%M", localtime())
agent_sample_rate_str = "__ASR_" + str(agent_sample_rate)
alt_sample_size_str = "_ALT_" + str(alt_sample_size)
info_file = date_time_str + agent_sample_rate_str + alt_sample_size_str + "__info.txt"
logger.enable_file_logging(date_time_str + agent_sample_rate_str + alt_sample_size_str + "__run.txt")
logger.enable_memory_logging()
logger.log_status("Constrained Estimation with agent sample rate of %s and alternatvie sample size %s\n" % \
(agent_sample_rate, alt_sample_size))
t1 = time()
SimulationState().set_current_time(2000)
self.nbs = SessionConfiguration().get_dataset_from_pool("neighborhood")
self.hhs = SessionConfiguration().get_dataset_from_pool('household')
depts, lambda_value = compute_lambda(self.nbs)
supply, vacancy_rate = compute_supply_and_vacancy_rate(self.nbs, depts, lambda_value)
self.nbs.set_values_of_one_attribute("supply", supply)
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset_pool.add_datasets_if_not_included({'vacancy_rate': vacancy_rate,
'sample_rate':agent_sample_rate
})
SessionConfiguration()["CLOSE"] = CLOSE
SessionConfiguration()['info_file'] = info_file
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(in_table_name="household_location_choice_model_specification")
#submodel_string = "workers"
seed(71) # was: seed(71,110)
self.model_name = "household_location_choice_model"
model = HouseholdLocationChoiceModelCreator().get_model(location_set=self.nbs,
submodel_string=submodel_string,
sampler = sampler,
estimation_size_agents = agent_sample_rate * 100/20,
# proportion of the agent set that should be used for the estimation,
#
sample_size_locations = alt_sample_size, # choice set size (includes current location)
compute_capacity_flag = True,
probabilities = "opus_core.mnl_probabilities",
choices = "urbansim.lottery_choices",
run_config = Resources({"capacity_string":"supply"}),
estimate_config = Resources({"capacity_string":"supply","compute_capacity_flag":True}))
#TODO: since households_for_estimation currently is the same as households, create_households_for_estimation
#becomes unnecesarry
#agent_set, agents_index_for_estimation = create_households_for_estimation(self.hhs, self.in_con)
agent_set = self.hhs; agents_index_for_estimation = arange(self.hhs.size())
self.result = model.estimate(self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops" ) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time()-t1) + " s")
def estimate(self,
spec_var=None,
spec_py=None,
submodel_string="workers",
agent_sample_rate=0.005,
alt_sample_size=None):
"""
"""
CLOSE = 0.001
sampler = "opus_core.samplers.weighted_sampler"
if alt_sample_size == None:
sampler = None
date_time_str = strftime("%Y_%m_%d__%H_%M", localtime())
agent_sample_rate_str = "__ASR_" + str(agent_sample_rate)
alt_sample_size_str = "_ALT_" + str(alt_sample_size)
info_file = date_time_str + agent_sample_rate_str + alt_sample_size_str + "__info.txt"
logger.enable_file_logging(date_time_str + agent_sample_rate_str +
alt_sample_size_str + "__run.txt")
logger.enable_memory_logging()
logger.log_status("Constrained Estimation with agent sample rate of %s and alternatvie sample size %s\n" % \
(agent_sample_rate, alt_sample_size))
t1 = time()
SimulationState().set_current_time(2000)
self.nbs = SessionConfiguration().get_dataset_from_pool("neighborhood")
self.hhs = SessionConfiguration().get_dataset_from_pool('household')
depts, lambda_value = compute_lambda(self.nbs)
supply, vacancy_rate = compute_supply_and_vacancy_rate(
self.nbs, depts, lambda_value)
self.nbs.set_values_of_one_attribute("supply", supply)
dataset_pool = SessionConfiguration().get_dataset_pool()
dataset_pool.add_datasets_if_not_included({
'vacancy_rate':
vacancy_rate,
'sample_rate':
agent_sample_rate
})
SessionConfiguration()["CLOSE"] = CLOSE
SessionConfiguration()['info_file'] = info_file
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(
in_table_name="household_location_choice_model_specification")
#submodel_string = "workers"
seed(71) # was: seed(71,110)
self.model_name = "household_location_choice_model"
model = HouseholdLocationChoiceModelCreator().get_model(
location_set=self.nbs,
submodel_string=submodel_string,
sampler=sampler,
estimation_size_agents=agent_sample_rate * 100 / 20,
# proportion of the agent set that should be used for the estimation,
#
sample_size_locations=
alt_sample_size, # choice set size (includes current location)
compute_capacity_flag=True,
probabilities="opus_core.mnl_probabilities",
choices="urbansim.lottery_choices",
run_config=Resources({"capacity_string": "supply"}),
estimate_config=Resources({
"capacity_string": "supply",
"compute_capacity_flag": True
}))
#TODO: since households_for_estimation currently is the same as households, create_households_for_estimation
#becomes unnecesarry
#agent_set, agents_index_for_estimation = create_households_for_estimation(self.hhs, self.in_con)
agent_set = self.hhs
agents_index_for_estimation = arange(self.hhs.size())
self.result = model.estimate(
self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops"
) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time() - t1) + " s")
def estimate(self, spec_var=None, spec_py=None,
movers_index = None,
submodel_string = "",
alt_sample_size=None,
sampler = "opus_core.samplers.weighted_sampler",
weight_string = "supply",
aggregate_demand = False,
submarket_definition = ('zone', 'building_type_id'),
sample_size_from_each_stratum = 50
):
"""
"""
t1 = time()
SimulationState().set_current_time(2000)
dataset_pool=SessionConfiguration().get_dataset_pool()
buildings = dataset_pool.get_dataset("building")
agent_set = dataset_pool.get_dataset('household')
#buildings.load_dataset()
submarket_geography = dataset_pool.get_dataset(submarket_definition[0])
intermediates = '[]'
if submarket_geography.dataset_name == 'zone':
intermediates = '[parcel]'
elif submarket_geography.dataset_name == 'faz':
intermediates = '[zone, parcel]'
elif submarket_geography.dataset_name == 'large_area':
intermediates = '[faz, zone, parcel]'
submarket_id_expression = 'building.disaggregate(%s.%s, intermediates=%s) * 100' % \
(submarket_geography.dataset_name, submarket_geography.get_id_name()[0],
intermediates)
submarket_variables = ['%s=numpy.ceil(submarket.submarket_id / 100)' % submarket_geography.get_id_name()[0]]
if submarket_definition[1] == 'residential_building_type_id':
set_residential_building_types(dataset_pool.get_dataset("building_type"), dataset_pool.get_dataset("building"))
if submarket_definition[1] != '':
submarket_id_expression = submarket_id_expression + ' + building.%s' % submarket_definition[1]
submarket_variables.append(submarket_definition[1] + '=submarket.submarket_id % 100' )
submarkets = define_submarket(buildings,
submarket_id_expression,
#"urbansim_parcel.building.zone_id*100 + building.residential_building_type_id",
#"building.disaggregate(faz.large_area_id, intermediates=[zone, parcel]) * 100 + building.residential_building_type_id",
compute_variables=submarket_variables + [
"residential_units=submarket.aggregate(building.residential_units)",
"number_of_buildings_with_non_zero_units=submarket.aggregate(building.residential_units > 0 )",
"number_of_surveyed_households=submarket.aggregate(household.household_id > 5000000, intermediates=[building])",
],
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'submarket.supply > 0',
#"psrc_parcel.building.large_area_id*100 + building.residential_building_type_id",
#compute_variables=['residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id / 100)']
#"psrc_parcel.building.large_area_id",
#compute_variables=[#'residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id)']
)
dataset_pool.add_datasets_if_not_included({'submarket':submarkets})
compute_lambda_and_supply(buildings, agent_set, movers_index, submarkets)
submarket_filter = 'submarket.supply > 0'
if submarket_filter is not None:
from numpy import logical_not
submarkets.remove_elements(index= where( logical_not(submarkets.compute_variables(submarket_filter)) )[0])
submarkets.touch_attribute(submarkets.get_id_name()[0])
buildings.touch_attribute(submarkets.get_id_name()[0])
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(type='sql_storage',
storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(in_table_name="household_location_choice_model_specification")
self.model_name = "household_location_choice_model"
agent_set, agents_index_for_estimation = get_households_for_estimation(agent_set,
AttributeCache(),
"households_for_estimation",
exclude_condition="household.disaggregate(submarket.submarket_id, intermediates=[building])<=0",
)
agent_set.compute_variables("submarket_id=household.disaggregate(building.submarket_id)")
agent_sample_rate = agents_index_for_estimation.size / float(movers_index.size)
dataset_pool.add_datasets_if_not_included({'sample_rate': agent_sample_rate })
if aggregate_demand:
location_set = buildings
aggregate_dataset = 'submarket'
#weight_string = 'inv_submarket_supply = 1.0 / (building.disaggregate(submarket.number_of_agents(building))).astype(float32) * (building.disaggregate(submarket.submarket_id) > 0)'
#weight_string = 'submarket_supply = (building.disaggregate(submarket.supply) > 0).astype(int32)'
#weight_string = 'submarket_supply = building.disaggregate(submarket.supply) * (building.disaggregate(submarket.submarket_id) > 0).astype(float32)'
else:
location_set = submarkets
aggregate_dataset = None
#weight_string = 'supply'
model = HouseholdLocationChoiceModelCreator().get_model(location_set=location_set,
#location_set=submarkets,
#filter = 'building.disaggregate(submarket.submarket_id) > 0',
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'building.disaggregate(numpy.logical_and(submarket.number_of_buildings_with_non_zero_units > 5000, submarket.number_of_surveyed_households > 0))',
submodel_string=submodel_string,
sampler = sampler,
#estimation_size_agents = agent_sample_rate * 100/20,
# proportion of the agent set that should be used for the estimation
sample_size_locations = alt_sample_size,
#sample_proportion_locations = 1.0/1000,
# choice set size (includes current location)
compute_capacity_flag = True,
probabilities = "opus_core.mnl_probabilities",
choices = "urbansim.lottery_choices",
#run_config = Resources({"capacity_string":"supply"}),
estimate_config = Resources({"capacity_string":"supply",
"weights_for_estimation_string":weight_string,
"aggregate_to_dataset":aggregate_dataset,
"stratum": "building.disaggregate(submarket.submarket_id)",
"sample_size_from_each_stratum": sample_size_from_each_stratum,
#"index2":where(submarkets.compute_variables('submarket.number_of_surveyed_households > 0'))[0],
#"sample_rate": 1.0/5000,
#"sample_size_from_chosen_stratum": 0,
"include_chosen_choice": True
}))
# was dataset_pool.add_datasets_if_not_included({'sample_rate':agent_sample_rate})
self.result = model.estimate(self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops" ) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time()-t1) + " s")
def estimate(self,
spec_var=None,
spec_py=None,
movers_index=None,
submodel_string="",
alt_sample_size=None,
sampler="opus_core.samplers.weighted_sampler",
weight_string="supply",
aggregate_demand=False,
submarket_definition=('zone', 'building_type_id'),
sample_size_from_each_stratum=50):
"""
"""
t1 = time()
SimulationState().set_current_time(2000)
dataset_pool = SessionConfiguration().get_dataset_pool()
buildings = dataset_pool.get_dataset("building")
agent_set = dataset_pool.get_dataset('household')
#buildings.load_dataset()
submarket_geography = dataset_pool.get_dataset(submarket_definition[0])
intermediates = '[]'
if submarket_geography.dataset_name == 'zone':
intermediates = '[parcel]'
elif submarket_geography.dataset_name == 'faz':
intermediates = '[zone, parcel]'
elif submarket_geography.dataset_name == 'large_area':
intermediates = '[faz, zone, parcel]'
submarket_id_expression = 'building.disaggregate(%s.%s, intermediates=%s) * 100' % \
(submarket_geography.dataset_name, submarket_geography.get_id_name()[0],
intermediates)
submarket_variables = [
'%s=numpy.ceil(submarket.submarket_id / 100)' %
submarket_geography.get_id_name()[0]
]
if submarket_definition[1] == 'residential_building_type_id':
set_residential_building_types(
dataset_pool.get_dataset("building_type"),
dataset_pool.get_dataset("building"))
if submarket_definition[1] != '':
submarket_id_expression = submarket_id_expression + ' + building.%s' % submarket_definition[
1]
submarket_variables.append(submarket_definition[1] +
'=submarket.submarket_id % 100')
submarkets = define_submarket(
buildings,
submarket_id_expression,
#"urbansim_parcel.building.zone_id*100 + building.residential_building_type_id",
#"building.disaggregate(faz.large_area_id, intermediates=[zone, parcel]) * 100 + building.residential_building_type_id",
compute_variables=submarket_variables + [
"residential_units=submarket.aggregate(building.residential_units)",
"number_of_buildings_with_non_zero_units=submarket.aggregate(building.residential_units > 0 )",
"number_of_surveyed_households=submarket.aggregate(household.household_id > 5000000, intermediates=[building])",
],
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'submarket.supply > 0',
#"psrc_parcel.building.large_area_id*100 + building.residential_building_type_id",
#compute_variables=['residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id / 100)']
#"psrc_parcel.building.large_area_id",
#compute_variables=[#'residential_building_type_id=submarket.submarket_id % 100',
#'large_area_id=numpy.ceil(submarket.submarket_id)']
)
dataset_pool.add_datasets_if_not_included({'submarket': submarkets})
compute_lambda_and_supply(buildings, agent_set, movers_index,
submarkets)
submarket_filter = 'submarket.supply > 0'
if submarket_filter is not None:
from numpy import logical_not
submarkets.remove_elements(index=where(
logical_not(submarkets.compute_variables(submarket_filter)))
[0])
submarkets.touch_attribute(submarkets.get_id_name()[0])
buildings.touch_attribute(submarkets.get_id_name()[0])
if self.save_estimation_results:
out_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.out_con)
if spec_py is not None:
reload(spec_py)
spec_var = spec_py.specification
if spec_var is not None:
self.specification = load_specification_from_dictionary(spec_var)
else:
in_storage = StorageFactory().build_storage_for_dataset(
type='sql_storage', storage_location=self.in_con)
self.specification = EquationSpecification(in_storage=in_storage)
self.specification.load(
in_table_name="household_location_choice_model_specification")
self.model_name = "household_location_choice_model"
agent_set, agents_index_for_estimation = get_households_for_estimation(
agent_set,
AttributeCache(),
"households_for_estimation",
exclude_condition=
"household.disaggregate(submarket.submarket_id, intermediates=[building])<=0",
)
agent_set.compute_variables(
"submarket_id=household.disaggregate(building.submarket_id)")
agent_sample_rate = agents_index_for_estimation.size / float(
movers_index.size)
dataset_pool.add_datasets_if_not_included(
{'sample_rate': agent_sample_rate})
if aggregate_demand:
location_set = buildings
aggregate_dataset = 'submarket'
#weight_string = 'inv_submarket_supply = 1.0 / (building.disaggregate(submarket.number_of_agents(building))).astype(float32) * (building.disaggregate(submarket.submarket_id) > 0)'
#weight_string = 'submarket_supply = (building.disaggregate(submarket.supply) > 0).astype(int32)'
#weight_string = 'submarket_supply = building.disaggregate(submarket.supply) * (building.disaggregate(submarket.submarket_id) > 0).astype(float32)'
else:
location_set = submarkets
aggregate_dataset = None
#weight_string = 'supply'
model = HouseholdLocationChoiceModelCreator().get_model(
location_set=location_set,
#location_set=submarkets,
#filter = 'building.disaggregate(submarket.submarket_id) > 0',
#filter = 'numpy.logical_and(submarket.number_of_surveyed_households > 0, submarket.residential_units>0)',
#filter = 'building.disaggregate(numpy.logical_and(submarket.number_of_buildings_with_non_zero_units > 5000, submarket.number_of_surveyed_households > 0))',
submodel_string=submodel_string,
sampler=sampler,
#estimation_size_agents = agent_sample_rate * 100/20,
# proportion of the agent set that should be used for the estimation
sample_size_locations=alt_sample_size,
#sample_proportion_locations = 1.0/1000,
# choice set size (includes current location)
compute_capacity_flag=True,
probabilities="opus_core.mnl_probabilities",
choices="urbansim.lottery_choices",
#run_config = Resources({"capacity_string":"supply"}),
estimate_config=Resources({
"capacity_string": "supply",
"weights_for_estimation_string": weight_string,
"aggregate_to_dataset": aggregate_dataset,
"stratum": "building.disaggregate(submarket.submarket_id)",
"sample_size_from_each_stratum": sample_size_from_each_stratum,
#"index2":where(submarkets.compute_variables('submarket.number_of_surveyed_households > 0'))[0],
#"sample_rate": 1.0/5000,
#"sample_size_from_chosen_stratum": 0,
"include_chosen_choice": True
}))
# was dataset_pool.add_datasets_if_not_included({'sample_rate':agent_sample_rate})
self.result = model.estimate(
self.specification,
agent_set=agent_set,
agents_index=agents_index_for_estimation,
debuglevel=self.debuglevel,
procedure="urbansim.constrain_estimation_bhhh_two_loops"
) #"urbansim.constrain_estimation_bhhh"
#save estimation results
if self.save_estimation_results:
self.save_results(out_storage)
logger.log_status("Estimation done. " + str(time() - t1) + " s")
