def ndsum(input, labels, index=None):
""" extend scipy.ndimage.sum to handle labels with multi-array
index argument is not used
e.g.
input = array([3, 7, 4, 6, 2, 5 ])
attr_a = array([0, 0, 1, 0, 1, 1])
attr_b = array([3, 1, 2, 1, 2, 0])
result = ndsum(input, labels=column_stack([attr_a, attr_b]))
print result
>>> (array([13, 3, 5, 6]), (array([0, 0, 1, 1]), array([1, 3, 0, 2])) )
"""
if labels is None or not isinstance(labels, ndarray):
return sum(input, labels=labels, index=index)
assert input.size == labels.shape[0]
#labels = column_stack(labels)
hash_table = {}
def hashlabel(label):
hash_value = djbhash(label)
hash_table.update({hash_value:label})
return hash_value
labels_hash = array(map(hashlabel, labels)).astype("int32")
index = array(hash_table.keys()).astype("int32")
value = array(hash_table.values())
result = sum(input, labels=labels_hash, index=index)
def get_demand_for_submodel(self, submodel=0):
"""Return aggregated probabilities for each location for the submodel."""
probs = self.upc_sequence.get_probabilities()
demand = ndimage.sum(probs.ravel().astype("float32"),
labels=self.run_config["index"].ravel() + 1,
index=arange(self.choice_set.size()) + 1)
return demand
def compute_demand(self, probabilities):
"""sums probabilities for each alternative and adds it to the demand attribute of the choice set.
"""
demand = ndimage.sum(
probabilities.ravel().astype("float32"),
labels=self.run_config["index"].ravel() + 1,
index=arange(self.choice_set.size()) + 1,
)
demand_attr = self.run_config.get("demand_string")
self.choice_set.modify_attribute(name=demand_attr, data=self.choice_set.get_attribute(demand_attr) + demand)
def aggregate(self, values, aggregate_from, aggregate_to, intermediates=[]):
dataset_pool = self._setup_environment(self.cache_set[0], self.get_base_year())
ds_from = dataset_pool.get_dataset(aggregate_from)
dataset_names = intermediates + [aggregate_to]
new_values = values.copy()
for dataset_name in dataset_names:
aggr_values = new_values.copy()
ds_to = dataset_pool.get_dataset(dataset_name)
ids = ds_from.get_attribute(ds_to.get_id_name()[0])
new_values = zeros((ds_to.size(), aggr_values.shape[1]), dtype=values.dtype)
for i in range(aggr_values.shape[1]):
new_values[:,i] = ndimage.sum(aggr_values[:,i], labels=ids, index=ds_to.get_id_attribute())
ds_from = ds_to
return (new_values, ds_to.get_id_attribute())
def get_category_and_frequency(agent_set, agent_category_definition,
choice_set, choice_category_definition,
agent_filter_attribute, category_inflating_factor,
dataset_pool):
agent_category_variable = []
for i in range( len(agent_category_definition) ):
agent_category_variable.append( VariableName(agent_category_definition[i]).get_alias() + \
'*%i' % category_inflating_factor**i )
if len(agent_category_variable) > 0:
agent_category_id = agent_set.compute_variables("agent_category_id=" + ' + '.join(agent_category_variable), dataset_pool=dataset_pool)
else:
agent_category_id = agent_set.get_id_attribute()
unique_agent_category_id = unique(agent_category_id)
choice_category_variable = []
for i in range( len(choice_category_definition) ):
#choice_category_variable.append( VariableName(choice_category_definition[i]).get_alias() + \
# '*%i' % category_inflating_factor**i )
choice_category_variable.append( choice_category_definition[i] + \
'*%i' % category_inflating_factor**i )
if len(choice_category_variable) > 0:
choice_category_id = choice_set.compute_variables("choice_category_id=" + ' + '.join( choice_category_variable), dataset_pool=dataset_pool)
agent_choice_category_id = agent_set.compute_variables('choice_category_id=%s.disaggregate(%s.choice_category_id)' % \
(agent_set.get_dataset_name(), choice_set.get_dataset_name()),
dataset_pool=dataset_pool)
else:
choice_category_id = choice_set.get_id_attribute()
agent_choice_category_id = agent_set.get_attribute(choice_set.get_id_name()[0])
unique_choice_category_id = unique(choice_category_id)
if agent_filter_attribute is not None and len(agent_filter_attribute) > 0:
agent_filter = agent_set.compute_variables(agent_filter_attribute, dataset_pool=dataset_pool)
else:
agent_filter = ones(agent_set.size(), dtype='bool')
frequency = zeros( (unique_agent_category_id.size, unique_choice_category_id.size), dtype="int32" )
for i in range(unique_agent_category_id.size): # iterate over agent category
is_agent_of_this_category = logical_and(agent_filter, agent_category_id == unique_agent_category_id[i]).astype("int32")
frequency[i,:] = ndimage.sum(is_agent_of_this_category, labels=agent_choice_category_id.astype("int32"), index=unique_choice_category_id.astype("int32"))
return frequency, unique_agent_category_id, unique_choice_category_id, agent_category_id, choice_category_id
def run(self, n=500,
realestate_dataset_name = 'building',
current_year=None,
occupied_spaces_variable="occupied_spaces",
total_spaces_variable="total_spaces",
minimum_spaces_attribute="minimum_spaces",
within_parcel_selection_weight_string=None,
within_parcel_selection_n=0,
within_parcel_selection_compete_among_types=False,
within_parcel_selection_threshold=75,
within_parcel_selection_MU_same_weight=False,
within_parcel_selection_transpose_interpcl_weight=True,
run_config=None,
debuglevel=0):
"""
run method of the Development Project Proposal Sampling Model
**Parameters**
**n** : int, sample size for each iteration
sample n proposals at a time, which are then evaluated one by one until the
target vacancies are satisfied or proposals are running out
**realestate_dataset_name** : string, name of real estate dataset
**current_year**: int, simulation year. If None, get value from SimulationState
**occupied_spaces_variable** : string, variable name for calculating how much spaces are currently occupied
It can either be a variable for real_estate dataset that returns
the amount spaces being occupied or a target_vacancy attribute
that contains the name of real_estate variables.
**total_spaces_variable** : string, variable name for calculating total existing spaces
**Returns**
**proposal_set** : indices to proposal_set that are accepted
**demolished_buildings** : buildings to be demolished for re-development
"""
self.accepted_proposals = []
self.demolished_buildings = [] #id of buildings to be demolished
if self.proposal_set.n <= 0:
logger.log_status("The size of proposal_set is 0; no proposals to consider, skipping DPPSM.")
return (self.proposal_set, self.demolished_buildings)
target_vacancy = self.dataset_pool.get_dataset('target_vacancy')
if current_year is None:
year = SimulationState().get_current_time()
else:
year = current_year
this_year_index = where(target_vacancy['year']==year)[0]
target_vacancy_for_this_year = DatasetSubset(target_vacancy, this_year_index)
if target_vacancy_for_this_year.size() == 0:
raise IOError, 'No target vacancy defined for year %s.' % year
## current_target_vacancy.target_attribute_name = 'target_vacancy_rate'
## each column provides a category for which a target vacancy is specified
self.column_names = list(set( target_vacancy.get_known_attribute_names() ) - \
set( [ target_vacancy.target_attribute_name,
'year', '_hidden_id_', minimum_spaces_attribute,
occupied_spaces_variable, total_spaces_variable
] )
)
self.column_names.sort(reverse=True)
## buildings table provides existing stocks
self.realestate_dataset = self.dataset_pool.get_dataset(realestate_dataset_name)
occupied_spaces_variables = [occupied_spaces_variable]
total_spaces_variables = [total_spaces_variable]
if occupied_spaces_variable in target_vacancy_for_this_year.get_known_attribute_names():
occupied_spaces_variables += unique(target_vacancy_for_this_year[occupied_spaces_variable]).tolist()
if total_spaces_variable in target_vacancy_for_this_year.get_known_attribute_names():
total_spaces_variables += unique(target_vacancy_for_this_year[total_spaces_variable]).tolist()
self._compute_variables_for_dataset_if_needed(self.realestate_dataset, self.column_names + occupied_spaces_variables + total_spaces_variables)
self._compute_variables_for_dataset_if_needed(self.proposal_component_set, self.column_names + total_spaces_variables)
self.proposal_set.compute_variables(["urbansim_parcel.development_project_proposal.number_of_components",
"urbansim_parcel.development_project_proposal.land_area_taken"],
dataset_pool=self.dataset_pool)
n_column = len(self.column_names)
self.column_names_index = {}
for iname in range(n_column):
self.column_names_index[self.column_names[iname]] = iname
target_vacancy_for_this_year.column_values = target_vacancy_for_this_year.get_multiple_attributes(self.column_names).reshape((-1, n_column))
self.realestate_dataset.column_values = self.realestate_dataset.get_multiple_attributes(self.column_names).reshape((-1, n_column))
self.proposal_component_set.column_values = self.proposal_component_set.get_multiple_attributes(self.column_names).reshape((-1, n_column))
#defaults, can be changed later by spaces_variable specified in target_vacancy rates
self.realestate_dataset.total_spaces = self.realestate_dataset[total_spaces_variable]
self.proposal_component_set.total_spaces = self.proposal_component_set[total_spaces_variable]
self.realestate_dataset.occupied_spaces = self.realestate_dataset[occupied_spaces_variable]
self.accounting = {}; self.logging = {}
#has_needed_components = zeros(self.proposal_set.size(), dtype='bool')
for index in range(target_vacancy_for_this_year.size()):
column_value = tuple(target_vacancy_for_this_year.column_values[index,:].tolist())
accounting = {'target_vacancy': target_vacancy_for_this_year[target_vacancy.target_attribute_name][index]}
if minimum_spaces_attribute in target_vacancy_for_this_year.get_known_attribute_names():
accounting['minimum_spaces'] = target_vacancy_for_this_year[minimum_spaces_attribute][index]
realestate_indexes = self.get_index_by_condition(self.realestate_dataset.column_values, column_value)
component_indexes = self.get_index_by_condition(self.proposal_component_set.column_values, column_value)
this_total_spaces_variable, this_occupied_spaces_variable = total_spaces_variable, occupied_spaces_variable
## total/occupied_spaces_variable can be specified either as a universal name for all realestate
## or in targe_vacancy_rate dataset for each vacancy category
if occupied_spaces_variable in target_vacancy_for_this_year.get_known_attribute_names():
this_occupied_spaces_variable = target_vacancy_for_this_year[occupied_spaces_variable][index]
self.realestate_dataset.occupied_spaces[realestate_indexes] = (self.realestate_dataset[this_occupied_spaces_variable][realestate_indexes]
).astype(self.realestate_dataset.occupied_spaces.dtype)
if total_spaces_variable in target_vacancy_for_this_year.get_known_attribute_names():
this_total_spaces_variable = target_vacancy_for_this_year[total_spaces_variable][index]
self.realestate_dataset.total_spaces[realestate_indexes] = (self.realestate_dataset[this_total_spaces_variable][realestate_indexes]
).astype(self.realestate_dataset.total_spaces.dtype)
self.proposal_component_set.total_spaces[component_indexes] = (self.proposal_component_set[this_total_spaces_variable][component_indexes]
).astype(self.proposal_component_set.total_spaces.dtype)
accounting["total_spaces_variable"] = this_total_spaces_variable
accounting["total_spaces"] = self.realestate_dataset.total_spaces[realestate_indexes].sum()
accounting["occupied_spaces_variable"] = this_occupied_spaces_variable
accounting["occupied_spaces"] = self.realestate_dataset.occupied_spaces[realestate_indexes].sum()
accounting["target_spaces"] = int( round( accounting["occupied_spaces"] /\
(1 - accounting["target_vacancy"])
) )
accounting["proposed_spaces"] = 0
accounting["demolished_spaces"] = 0
self.accounting[column_value] = accounting
if self._is_target_reached(column_value):
proposal_indexes = self.proposal_set.get_id_index(unique(self.proposal_component_set['proposal_id'][component_indexes]))
if n_column == 1:
comp_indexes = where(ndimage.sum(self.proposal_component_set[self.column_names[0]]==column_value[0],
labels=self.proposal_component_set['proposal_id'],
index=self.proposal_set.get_id_attribute()
) == self.proposal_set["number_of_components"])[0]
else:
comp_indexes = where(self.proposal_set["number_of_components"]==1)[0]
target_reached_prop_idx = intersect1d(proposal_indexes, comp_indexes)
self.weight[target_reached_prop_idx] = 0.0
self.proposal_set["status_id"][intersect1d(target_reached_prop_idx, where(self.proposal_set["status_id"]==self.proposal_set.id_tentative)[0])] = self.proposal_set.id_no_demand
## handle planned proposals: all proposals with status_id == is_planned
## and start_year == year are accepted
planned_proposal_indexes = where(logical_and(
self.proposal_set.get_attribute("status_id") == self.proposal_set.id_planned,
self.proposal_set.get_attribute("start_year") == year )
)[0]
logger.start_block("Processing %s planned proposals" % planned_proposal_indexes.size)
self.consider_proposals(planned_proposal_indexes, force_accepting=True)
logger.end_block()
if within_parcel_selection_n > 0:
logger.start_block("Selecting proposals within parcels (%s proposals per parcel)" % within_parcel_selection_n)
self.select_proposals_within_parcels(nmax=within_parcel_selection_n, weight_string=within_parcel_selection_weight_string,
compete_among_types=within_parcel_selection_compete_among_types,
filter_threshold=within_parcel_selection_threshold,
MU_same_weight=within_parcel_selection_MU_same_weight,
transpose_interpcl_weight=within_parcel_selection_transpose_interpcl_weight)
logger.end_block()
# consider proposals (in this order: proposed, tentative)
for status in [self.proposal_set.id_proposed, self.proposal_set.id_tentative]:
stat = (self.proposal_set.get_attribute("status_id") == status)
if stat.sum() == 0:
continue
logger.log_status("Sampling from %s eligible proposals of status %s." % (stat.sum(), status))
iteration = 0
while (not self._is_target_reached()):
## prevent proposals from being sampled for vacancy type whose target is reached
#for column_value in self.accounting.keys():
if self.weight[stat].sum() == 0.0:
logger.log_warning("Running out of proposals of status %s before vacancy targets are reached; there aren't any proposals with non-zero weight" % status)
break
available_indexes = where(logical_and(stat, self.weight > 0))[0]
sample_size = minimum(available_indexes.size, n)
sampled_proposal_indexes = probsample_noreplace(available_indexes, sample_size,
prob_array=self.weight[available_indexes],
return_index=False)
#sorted_sampled_indices = argsort(self.weight[sampled_proposal_indexes])
#self.consider_proposals(sampled_proposal_indexes[sorted_sampled_indices][::-1])
self.consider_proposals(sampled_proposal_indexes)
self.weight[sampled_proposal_indexes] = 0
iteration += 1
self._log_status()
# set status of accepted proposals to 'active'
self.proposal_set.modify_attribute(name="status_id",
data=self.proposal_set.id_active,
index=array(self.accepted_proposals, dtype='int32'))
# Code added by Jesse Ayers, MAG, 7/20/2009
# Get the active projects:
stat_id = self.proposal_set.get_attribute('status_id')
actv = where(stat_id==1)[0]
# Where there are active projects, compute the total_land_area_taken
# and store it on the development_project_proposals dataset
# so it can be used by the building_construction_model for the proper
# computation of units_proposed for those projects with velocity curves
if actv.size > 0:
total_land_area_taken_computed = self.proposal_set['land_area_taken']
self.proposal_set.modify_attribute('total_land_area_taken', total_land_area_taken_computed[actv], actv)
return (self.proposal_set, self.realestate_dataset.get_id_attribute()[self.demolished_buildings])
def select_proposals_within_parcels(self, nmax=2, weight_string=None, compete_among_types=False, filter_threshold=75,
MU_same_weight=False, transpose_interpcl_weight=True):
# Allow only nmax proposals per parcel in order to not disadvantage parcels with small amount of proposals.
# It takes proposals with the highest weights.
#parcels_with_proposals = unique(self.proposal_set['parcel_id'])
#parcel_set = self.dataset_pool.get_dataset('parcel')
if weight_string is not None:
within_parcel_weights = self.proposal_set.compute_variables([weight_string], dataset_pool=self.dataset_pool)
else:
within_parcel_weights = self.weight
egligible = logical_and(self.weight > 0,
self.proposal_set['status_id'] == self.proposal_set.id_tentative)
wegligible = where(egligible)[0]
if wegligible.size <=0:
return
#parcels_with_proposals = unique(self.proposal_set['parcel_id'][wegligible])
#min_type = {}
#egligible_proposals = {}
tobechosen_ind = ones(wegligible.size).astype('bool8')
if not compete_among_types:
for key in self.column_names:
utypes_all = unique(self.proposal_component_set[key])
categories = zeros(self.proposal_set.size(), dtype='int32')
for btype in utypes_all:
w = where(ndimage.sum(self.proposal_component_set[key] == btype,
labels=self.proposal_component_set['proposal_id'],
index=self.proposal_set.get_id_attribute()
) == self.proposal_set["number_of_components"])[0]
categories[w] = btype
# categories equal zero means mix-used type with components of different type
utypes = unique(categories[wegligible])
for value in utypes:
type_is_value_ind = categories[wegligible]==value
for i in range(nmax):
parcels_with_proposals = (unique(self.proposal_set['parcel_id'][wegligible][where(type_is_value_ind)])).astype(int32)
if parcels_with_proposals.size <= 0:
continue
labels = (self.proposal_set['parcel_id'][wegligible])*type_is_value_ind
chosen_prop = array(maximum_position(within_parcel_weights[wegligible],
labels=labels,
index=parcels_with_proposals)).flatten().astype(int32)
egligible[wegligible[chosen_prop]] = False
type_is_value_ind[chosen_prop] = False
else:
parcels_with_proposals = unique(self.proposal_set['parcel_id'][wegligible]).astype(int32)
max_prop = array(maximum_position(within_parcel_weights[wegligible],
labels=self.proposal_set['parcel_id'][wegligible],
index=parcels_with_proposals)).flatten().astype(int32)
max_value_by_parcel = within_parcel_weights[wegligible][max_prop]
incompetition = ones(wegligible.size, dtype='bool8')
incompetition[max_prop] = False
egligible[wegligible[max_prop]] = False
for i in range(nmax-1):
labels = (self.proposal_set['parcel_id'][wegligible])*incompetition
valid_parcels = where(in1d(parcels_with_proposals, self.proposal_set['parcel_id'][wegligible][where(incompetition)]))[0]
if valid_parcels.size <= 0:
break
chosen_prop = array(maximum_position(within_parcel_weights[wegligible],
labels=labels,
index=parcels_with_proposals[valid_parcels])).flatten().astype(int32)
percent = within_parcel_weights[wegligible][chosen_prop]/(max_value_by_parcel[valid_parcels]/100.0)
where_lower = where(in1d(self.proposal_set['parcel_id'][wegligible], parcels_with_proposals[valid_parcels][percent <= filter_threshold]))[0]
egligible[wegligible[setdiff1d(chosen_prop, where_lower)]] = False # proposals with egligible=True get eliminated, so we dont want to set it to False for the where_lower ones
incompetition[union1d(chosen_prop, where_lower)] = False
if incompetition.sum() <= 0:
break
self.proposal_set['status_id'][where(egligible)] = self.proposal_set.id_eliminated_in_within_parcel_selection
if MU_same_weight:
# Set weights of mix-use proposals within the same parcel to the same value
parcels = self.dataset_pool.get_dataset('parcel')
# parcels.compute_variables(['mu_ind = parcel.aggregate(numpy.logical_or(development_project_proposal_component.building_type_id==4, development_project_proposal_component.building_type_id==12) + numpy.logical_or(development_project_proposal_component.building_type_id==3, development_project_proposal_component.building_type_id==13), intermediates=[development_project_proposal])'],
# dataset_pool=self.dataset_pool)
# pcl_ids = parcels.get_id_attribute()[parcels['mu_ind'] > 1]
# is_mu = logical_and(logical_and(self.weight > 0,
# self.proposal_set['status_id'] == self.proposal_set.id_tentative),
# in1d(self.proposal_set['parcel_id'], pcl_ids))
# where_mu = where(is_mu)[0]
# if where_mu.size <= 0:
# return
# trans_weights = self.weight[where_mu]
# if transpose_interpcl_weight:
# trans_weights = log(trans_weights)
# pcl_idx = parcels.get_id_index(self.proposal_set['parcel_id'][where_mu])
# upcl_idx = unique(pcl_idx)
# weight_mean = array(ndimage_mean(trans_weights, labels=pcl_idx, index=upcl_idx))
# if transpose_interpcl_weight:
# weight_mean = exp(weight_mean)
# weight_mean_tmp = zeros(upcl_idx.max()+1).astype(weight_mean.dtype)
# weight_mean_tmp[upcl_idx]=weight_mean
# self.weight[where_mu]=weight_mean_tmp[pcl_idx]
self.proposal_set.compute_variables(['is_mfres = development_project_proposal.aggregate(numpy.logical_or(development_project_proposal_component.building_type_id==4, development_project_proposal_component.building_type_id==12))'],
dataset_pool=self.dataset_pool)
parcels.compute_variables(['mu_ind = (parcel.aggregate(development_project_proposal.is_mfres)>0) * (parcel.mix_split_id > 0)'],
dataset_pool=self.dataset_pool)
pcl_ids = parcels.get_id_attribute()[parcels['mu_ind'] > 0]
egligible_props = logical_and(self.weight > 0, logical_and(
self.proposal_set['status_id'] == self.proposal_set.id_tentative,
self.proposal_set['is_mfres']>0))
where_prop_to_modify = where(logical_and(egligible_props,
in1d(self.proposal_set['parcel_id'], pcl_ids)))[0]
if where_prop_to_modify.size <= 0:
return
upcl = unique(self.proposal_set['parcel_id'][where_prop_to_modify])
npcl_to_modify = int(upcl.size/10.0)
if npcl_to_modify == 0:
return
pcls_to_modify = sample_noreplace(upcl, npcl_to_modify)
where_prop_to_modify_final = where(logical_and(egligible_props,
in1d(self.proposal_set['parcel_id'], pcls_to_modify)))[0]
trans_weights = self.weight[where_prop_to_modify_final]
if transpose_interpcl_weight:
trans_weights = log(trans_weights)
#trans_weights = 1.2*trans_weights
if transpose_interpcl_weight:
trans_weights = exp(trans_weights)
self.weight[where_prop_to_modify_final] = trans_weights
return
def run(self, developmentproject_dataset, building_dataset,
label_attribute_names=["building_type_id", "zone_id"],
quantity_attribute_names = ["residential_units", "non_residential_sqft"]):
"""Modify buildings to reflect new development projects.
"""
project_labels = None
building_labels = None
if not developmentproject_dataset or developmentproject_dataset.size() == 0:
logger.log_warning("Empty development project dataset. Skip add_projects_to_buildings.")
return building_dataset
is_placed_project = ones(developmentproject_dataset.size(), dtype='bool')
for label_attribute in label_attribute_names:
project_label_attribute = developmentproject_dataset.get_attribute_as_column(label_attribute)
is_placed_project = logical_and(is_placed_project, project_label_attribute[:,0]>0)
building_lable_attribute = building_dataset.get_attribute_as_column(label_attribute)
if project_labels is None:
project_labels = project_label_attribute
else:
project_labels = column_stack((project_labels, project_label_attribute))
if building_labels is None:
building_labels = building_lable_attribute
else:
building_labels = column_stack((building_labels, building_lable_attribute))
max_digits = digits( row_stack((project_labels, building_labels)).max(axis=0) )
multipler = array([10**d for d in max_digits[1:] + [0]])
building_identifier = (building_labels * multipler).sum(axis=1)
project_identifier = (project_labels * multipler).sum(axis=1)
if not all(is_placed_project):
logger.log_warning("There are %s projects with %s less than 0; they are not being processed." % (logical_not(is_placed_project).sum(),
",".join(label_attribute_names)))
project_identifier = project_identifier[is_placed_project]
unique_project_identifier = unique(project_identifier)
for quantity_attribute in quantity_attribute_names:
developmentproject_quantity = developmentproject_dataset.get_attribute(quantity_attribute)[is_placed_project]
if developmentproject_quantity.sum() == 0: continue
quantity_sum = ndimage.sum(developmentproject_quantity, labels=project_identifier, index=unique_project_identifier)
for i in range(unique_project_identifier.size):
if quantity_sum[i] != 0:
this_identifier = unique_project_identifier[i]
this_label = []
remain = this_identifier
for m in multipler:
this_label.append(remain // m)
remain = remain % m
building_index = where(building_identifier==this_identifier)[0]
#assert building_index.size == 1
if building_index.size == 0:
logger.log_error("building with attribute (%s) = (%s) is not in building_dataset" % (label_attribute_names, this_label) )
continue
#for attribute in []:
#data = None
#building_dataset.add_elements(name=quantity_attribute, data = current_values+quantity_sum[i],
#index=building_index)
if building_index.size > 1:
logger.log_warning("There are more than 1 building with attributes (%s) = (%s)" % (label_attribute_names, this_label) )
building_index = building_index[0]
current_values = building_dataset.get_attribute_by_index(quantity_attribute, building_index)
building_dataset.modify_attribute(name=quantity_attribute, data = current_values+quantity_sum[i],
index=building_index)
return building_dataset
def get_demand_for_submodel(self, submodel=0):
"""Return aggregated probabilities for each location for the submodel."""
probs = self.upc_sequence.get_probabilities()
demand = ndimage.sum(probs.ravel().astype("float32"), labels=self.run_config["index"].ravel()+1,
index=arange(self.choice_set.size())+1)
return demand
def run(self, n=500, run_config=None, current_year=None, debuglevel=0):
"""
n - sample n proposals at a time, evaluate them one by one
"""
self.demolished_buildings = array([], dtype="int32") # id of buildings to be demolished
if current_year is None:
current_year = SimulationState().get_current_time()
if not self.positive_proposals:
logger.log_status("Proposal Set size <= 0, no proposals to consider, skipping DPPSM.")
return (self.proposal_set, self.demolished_buildings)
self.proposal_component_set.compute_variables(
[
"urbansim_parcel.development_project_proposal_component.units_proposed",
"urbansim_parcel.development_project_proposal_component.is_residential",
],
dataset_pool=self.dataset_pool,
)
self.proposal_set.compute_variables(
[
"urbansim_parcel.development_project_proposal.number_of_components",
"zone_id=development_project_proposal.disaggregate(parcel.zone_id)",
#'occurence_frequency = development_project_proposal.disaggregate(development_template.sample_size)'
],
dataset_pool=self.dataset_pool,
)
buildings = self.dataset_pool.get_dataset("building")
buildings.compute_variables(
[
"occupied_units_for_jobs = urbansim_parcel.building.occupied_spaces",
"units_for_jobs = urbansim_parcel.building.total_spaces",
"occupied_residential_units = urbansim_parcel.building.number_of_households",
# "urbansim_parcel.building.existing_units",
"urbansim_parcel.building.is_residential",
],
dataset_pool=self.dataset_pool,
)
## define unit_name by whether a building is residential or not (with is_residential attribute)
## if it is non-residential (0), count units by number of job spaces (units_for_jobs)
## if it is residential (1), count units by residenital units
self.unit_name = array(["units_for_jobs", "residential_units"])
target_vacancy = self.dataset_pool.get_dataset("target_vacancy")
target_vacancy.compute_variables(
["is_residential = target_vacancy.disaggregate(building_type.is_residential)"],
dataset_pool=self.dataset_pool,
)
# This try-except block checks to see if the object has a subarea_id_name,
# if it does, it calculates the vacancy rates by subarea_id_name
try:
# Check for subarea_id_name in target_vacancies dataset
# if it is present, vacancy rates are specified by subarea_id_name
# if it is not, vacancy rates are specified region wide
target_vacancy.load_dataset()
if self.subarea_id_name in target_vacancy.get_attribute_names():
current_target_vacancy_this_year = DatasetSubset(
target_vacancy, index=where(target_vacancy.get_attribute("year") == current_year)[0]
)
current_target_vacancy = DatasetSubset(
current_target_vacancy_this_year,
index=where(current_target_vacancy_this_year.get_attribute(self.subarea_id_name) == self.area_id)[
0
],
)
else:
current_target_vacancy = DatasetSubset(
target_vacancy, index=where(target_vacancy.get_attribute("year") == current_year)[0]
)
except AttributeError:
# vacancy rates are specified region wide:
current_target_vacancy = DatasetSubset(
target_vacancy, index=where(target_vacancy.get_attribute("year") == current_year)[0]
)
if current_target_vacancy.size() == 0:
raise IOError, "No target vacancy defined for year %s." % current_year
self.existing_units = {} # total existing units by land_use type
self.occupied_units = {} # total occupied units by land_use type
self.proposed_units = {} # total proposed units by land_use type
self.demolished_units = {} # total (to be) demolished units by land_use type
components_building_type_ids = self.proposal_component_set.get_attribute("building_type_id").astype("int32")
proposal_ids = self.proposal_set.get_id_attribute()
proposal_ids_in_component_set = self.proposal_component_set.get_attribute("proposal_id")
all_units_proposed = self.proposal_component_set.get_attribute("units_proposed")
number_of_components_in_proposals = self.proposal_set.get_attribute("number_of_components")
self.accepting_proposals = zeros(
current_target_vacancy.get_attribute("building_type_id").max() + 1, dtype="bool8"
) # whether accepting new proposals, for each building type
self.accepted_proposals = [] # index of accepted proposals
self.target_vacancies = {}
tv_building_types = current_target_vacancy.get_attribute("building_type_id")
tv_rate = current_target_vacancy.get_attribute("target_vacancy_rate")
for itype in range(tv_building_types.size):
self.target_vacancies[tv_building_types[itype]] = tv_rate[itype]
self.check_vacancy_rates(
current_target_vacancy
) # initialize self.accepting_proposal based on current vacancy rate
sqft_per_job = self.dataset_pool.get_dataset("building_sqft_per_job")
zones_of_proposals = self.proposal_set.get_attribute("zone_id")
self.building_sqft_per_job_table = sqft_per_job.get_building_sqft_as_table(
zones_of_proposals.max(), tv_building_types.max()
)
# consider only those proposals that have all components of accepted type and sum of proposed units > 0
is_accepted_type = self.accepting_proposals[components_building_type_ids]
sum_is_accepted_type_over_proposals = array(
ndimage.sum(is_accepted_type, labels=proposal_ids_in_component_set, index=proposal_ids)
)
sum_of_units_proposed = array(
ndimage.sum(all_units_proposed, labels=proposal_ids_in_component_set, index=proposal_ids)
)
is_proposal_eligible = logical_and(
sum_is_accepted_type_over_proposals == number_of_components_in_proposals, sum_of_units_proposed > 0
)
is_proposal_eligible = logical_and(
is_proposal_eligible, self.proposal_set.get_attribute("start_year") == current_year
)
## handle planned proposals: all proposals with status_id == is_planned
## and start_year == current_year are accepted
planned_proposal_indexes = where(
logical_and(
self.proposal_set.get_attribute("status_id") == self.proposal_set.id_planned,
self.proposal_set.get_attribute("start_year") == current_year,
)
)[0]
self.consider_proposals(planned_proposal_indexes, force_accepting=True)
# consider proposals (in this order: planned, proposed, tentative)
for status in [self.proposal_set.id_proposed, self.proposal_set.id_tentative]:
idx = where(logical_and(self.proposal_set.get_attribute("status_id") == status, is_proposal_eligible))[0]
if idx.size <= 0:
continue
logger.log_status("Sampling from %s eligible proposals with status %s." % (idx.size, status))
while True in self.accepting_proposals:
if self.weight[idx].sum() == 0.0:
logger.log_warning("Running out of proposals; there aren't any proposals with non-zero weight")
break
idx = idx[self.weight[idx] > 0]
n = minimum(idx.size, n)
sampled_proposal_indexes = probsample_noreplace(
proposal_ids[idx],
n,
prob_array=(self.weight[idx] / float(self.weight[idx].sum())),
exclude_index=None,
return_index=True,
)
self.consider_proposals(arange(self.proposal_set.size())[idx[sampled_proposal_indexes]])
self.weight[idx[sampled_proposal_indexes]] = 0
# set status of accepted proposals to 'active'
self.proposal_set.modify_attribute(
name="status_id", data=self.proposal_set.id_active, index=array(self.accepted_proposals, dtype="int32")
)
building_types = self.dataset_pool.get_dataset("building_type")
logger.log_status("Status of %s development proposals set to active." % len(self.accepted_proposals))
logger.log_status("Target/existing vacancy rates (reached using eligible proposals) by building type:")
for type_id in self.existing_units.keys():
units_stock = self._get_units_stock(type_id)
vr = self._get_vacancy_rates(type_id)
## units = residential_units if building_type is residential
## units = number of job spaces if building_type is non-residential
logger.log_status(
"""%(type_id)s[%(type_name)s]: %(vr)s = ((existing_units:%(existing_units)s +
units_proposed:%(units_proposed)s - units_to_be_demolished:%(units_demolished)s)
- units_occupied:%(units_occupied)s) / units_stock:%(units_stock)s"""
% {
"type_id": type_id,
"type_name": building_types.get_attribute_by_id("building_type_name", type_id),
"vr": vr,
"existing_units": int(self.existing_units[type_id]),
"units_occupied": int(self.occupied_units[type_id]),
"units_proposed": int(self.proposed_units[type_id]),
"units_demolished": int(self.demolished_units[type_id]),
"units_stock": int(units_stock),
}
)
# Code added by Jesse Ayers, MAG, 7/20/2009
# Get the active projects:
stat_id = self.proposal_set.get_attribute("status_id")
actv = where(stat_id == 1)[0]
# Where there are active projects, compute the total_land_area_taken
# and store it on the development_project_proposals dataset
# so it can be used by the building_construction_model for the proper
# computation of units_proposed for those projects with velocity curves
if actv.size > 0:
total_land_area_taken_computed = self.proposal_set.get_attribute(
"urbansim_parcel.development_project_proposal.land_area_taken"
)
self.proposal_set.modify_attribute("total_land_area_taken", total_land_area_taken_computed[actv], actv)
return (self.proposal_set, self.demolished_buildings)
def consider_proposals(self, proposal_indexes, force_accepting=False):
proposals_parcel_ids = self.proposal_set.get_attribute("parcel_id")
components_building_type_ids = self.proposal_component_set.get_attribute("building_type_id").astype("int32")
proposal_ids = self.proposal_set.get_id_attribute()
proposal_ids_in_component_set = self.proposal_component_set.get_attribute("proposal_id")
all_units_proposed = self.proposal_component_set.get_attribute("units_proposed")
is_component_residential = self.proposal_component_set.get_attribute("is_residential")
number_of_components_in_proposals = self.proposal_set.get_attribute("number_of_components")
zones_of_proposals = self.proposal_set.get_attribute("zone_id")
is_proposal_rejected = zeros(proposal_indexes.size, dtype=bool8)
proposal_site = proposals_parcel_ids[proposal_indexes]
is_redevelopment = self.proposal_set.get_attribute_by_index("is_redevelopment", proposal_indexes)
buildings = self.dataset_pool.get_dataset("building")
building_site = buildings.get_attribute("parcel_id")
# building_existing_units = buildings.get_attribute("existing_units")
is_residential = buildings.get_attribute("is_residential")
building_type_ids = buildings.get_attribute("building_type_id")
building_ids = buildings.get_id_attribute()
for i in range(proposal_indexes.size):
if not (True in self.accepting_proposals) and not (force_accepting):
# if none of the types is accepting_proposals, exit
# this is put in the loop to check if the last accepted proposal has sufficed
# the target vacancy rates for all types
return
if is_proposal_rejected[i]:
continue
proposal_index = proposal_indexes[i] # consider 1 proposed project at a time
proposal_index_in_component_set = where(proposal_ids_in_component_set == proposal_ids[proposal_index])[0]
units_proposed = all_units_proposed[proposal_index_in_component_set]
component_types = components_building_type_ids[proposal_index_in_component_set]
is_this_component_residential = is_component_residential[proposal_index_in_component_set]
this_site = proposal_site[i]
if is_redevelopment[i]: # redevelopment proposal
affected_building_index = where(building_site == this_site)[0]
for this_building in affected_building_index:
this_building_type = building_type_ids[this_building]
if this_building_type in self.existing_units.keys():
_unit_name = self.unit_name[is_residential[this_building]]
self.demolished_units[this_building_type] += buildings.get_attribute(_unit_name)[
this_building
] # demolish affected buildings
self.demolished_buildings = concatenate(
(self.demolished_buildings, array([building_ids[this_building]]))
)
# self.occupied_units[type_id] = buildings.get_attribute("occupied_%s" % unit_name)[is_matched_type].astype("float32").sum()
for itype_id in range(component_types.size): #
# this loop is only needed when a proposal could provide units of more than 1 generic building types
type_id = component_types[itype_id]
if is_this_component_residential[itype_id]:
self.proposed_units[type_id] += units_proposed[itype_id]
else: # translate from building_sqft to number of job spaces
self.proposed_units[type_id] += (
units_proposed[itype_id]
/ self.building_sqft_per_job_table[zones_of_proposals[proposal_indexes[i]], type_id]
)
if not force_accepting:
## consider whether target vacancy rates have been achieved if not force_accepting
units_stock = self._get_units_stock(type_id)
vr = self._get_vacancy_rates(type_id)
if vr >= self.target_vacancies[type_id]:
## not accepting proposals of this type
self.accepting_proposals[type_id] = False
## reject all proposals to be processed that have one of the components of this type
consider_idx = proposal_indexes[(i + 1) : proposal_indexes.size]
if consider_idx.size > 0:
is_accepted_type = self.accepting_proposals[components_building_type_ids]
sum_is_accepted_type_over_proposals = array(
ndimage.sum(
is_accepted_type,
labels=proposal_ids_in_component_set,
index=proposal_ids[consider_idx],
)
)
is_rejected_indices = where(
sum_is_accepted_type_over_proposals < number_of_components_in_proposals[consider_idx]
)[0]
is_proposal_rejected[arange((i + 1), proposal_indexes.size)[is_rejected_indices]] = True
self.weight[consider_idx[is_rejected_indices]] = 0.0
if not is_proposal_rejected[i]:
# proposal accepted
self.accepted_proposals.append(proposal_index)
# reject all pending proposals for this site (1 site can accept only 1 proposal at any 1 given year)
is_proposal_rejected[proposal_site == this_site] = True
# don't consider proposals for this site in future sampling (in this year's developer model)
self.weight[proposals_parcel_ids == this_site] = 0.0
# if all proposals in proposal_indexes have being rejected, return
if is_proposal_rejected.sum() == is_proposal_rejected.size:
return
def create_prediction_success_table(self,
summarize_by=None,
predicted_choice_id_name=None,
predicted_choice_id_prefix="predicted_",
log_to_file=None,
force_predict=True):
agents = self.get_agent_set()
choices = self.get_choice_set()
choice_id_name = choices.get_id_name()[0]
if self.agents_index_for_prediction is not None:
agents_index = self.agents_index_for_prediction
else:
agents_index = self.get_agent_set_index()
if predicted_choice_id_name is None or len(predicted_choice_id_name) == 0:
predicted_choice_id_name = predicted_choice_id_prefix + choice_id_name
if force_predict or (predicted_choice_id_name not in agents.get_known_attribute_names()):
if not self.predict(predicted_choice_id_name=predicted_choice_id_name,
agents_index=agents_index
):
logger.log_error("Failed to run simulation for prediction; unable to create prediction success table.")
return
if log_to_file is not None and len(log_to_file) > 0:
logger.enable_file_logging(log_to_file)
## by default, compare predicted choice with observed choice
## this is not feasible for location choice model, where the
## alternative set is too large to be useful
if summarize_by is None:
summarize_by = "%s.%s" % (agents.dataset_name, choice_id_name)
summarize_dataset_name = VariableName(summarize_by).get_dataset_name()
if summarize_dataset_name == choices.dataset_name:
summary_id = choices.compute_variables(summarize_by)
chosen_choice_id = agents.get_attribute_by_index(choices.get_id_name()[0], agents_index)
predicted_choice_id = agents.get_attribute_by_index(predicted_choice_id_name, agents_index)
chosen_choice_index = choices.get_id_index(chosen_choice_id)
predicted_choice_index = choices.try_get_id_index(predicted_choice_id)
chosen_summary_id = summary_id[chosen_choice_index]
predicted_summary_id = summary_id[predicted_choice_index]
unique_summary_id = unique(summary_id)
elif summarize_dataset_name == agents.dataset_name:
chosen_summary_id = agents.compute_variables(summarize_by)[agents_index]
chosen_choice_id = agents.get_attribute(choice_id_name).copy()
predicted_choice_id = agents.get_attribute(predicted_choice_id_name)
agents.modify_attribute(name=choice_id_name, data=predicted_choice_id)
predicted_summary_id = agents.compute_variables(summarize_by)[agents_index]
agents.modify_attribute(name=choice_id_name, data=chosen_choice_id)
unique_summary_id = unique( concatenate((chosen_summary_id, predicted_summary_id)) )
else:
logger.log_error("summarize_by expression '%s' is specified for dataset %s, which is neither the choice_set '%s' nor the agent_set '%s'."
% (summarize_by, summarize_dataset_name, choices.dataset_name, agents.dataset_name))
return False
unique_nonneg_summary_id = unique_summary_id[unique_summary_id >= 0]
# observed on row, predicted on column
prediction_matrix = zeros( (unique_nonneg_summary_id.size, unique_nonneg_summary_id.size), dtype="int32" )
def _convert_array_to_tab_delimited_string(an_array):
from numpy import dtype
if an_array.dtype == dtype('f'):
return "\t".join(["%5.4f" % item for item in an_array])
return "\t".join([str(item) for item in an_array])
logger.log_status("Observed_id\tSuccess_rate\t%s" % \
_convert_array_to_tab_delimited_string(unique_nonneg_summary_id) )
i = 0
total_correct = 0
success_rate = zeros( unique_nonneg_summary_id.size, dtype="float32" )
for observed_id in unique_nonneg_summary_id:
predicted_id = predicted_summary_id[chosen_summary_id==observed_id]
prediction_matrix[i] = ndimage.sum(ones(predicted_id.size), labels=predicted_id, index=unique_nonneg_summary_id )
if prediction_matrix[i].sum() > 0:
if prediction_matrix[i].sum() > 0:
success_rate[i] = float(prediction_matrix[i, i]) / prediction_matrix[i].sum()
total_correct = total_correct + prediction_matrix[i, i]
else:
success_rate[i] = 0
logger.log_status("%s\t\t%5.4f\t\t%s" % (observed_id, success_rate[i],
_convert_array_to_tab_delimited_string(prediction_matrix[i]) ) )
i+=1
success_rate2 = zeros( i, dtype="float32" )
for j in range(i):
if prediction_matrix[j, :].sum() > 0:
success_rate2[j]=float(prediction_matrix[:,j].sum()) / prediction_matrix[j, :].sum()
else:
success_rate2[j]=0
logger.log_status("%s\t\t%s\t\t%s" % (' ', ' ',
_convert_array_to_tab_delimited_string( success_rate2 ) ))
logger.log_status("\nTotal success rate: %5.4f" % (total_correct/float(prediction_matrix.sum())))
logger.disable_file_logging(filename=log_to_file)
def create_prediction_success_table(self,
summarize_by=None,
predicted_choice_id_name=None,
predicted_choice_id_prefix="predicted_",
log_to_file=None,
force_predict=True):
agents = self.get_agent_set()
choices = self.get_choice_set()
choice_id_name = choices.get_id_name()[0]
if self.agents_index_for_prediction is not None:
agents_index = self.agents_index_for_prediction
else:
agents_index = self.get_agent_set_index()
if predicted_choice_id_name is None or len(predicted_choice_id_name) == 0:
predicted_choice_id_name = predicted_choice_id_prefix + choice_id_name
if force_predict or (predicted_choice_id_name not in agents.get_known_attribute_names()):
if not self.predict(predicted_choice_id_name=predicted_choice_id_name,
agents_index=agents_index
):
logger.log_error("Failed to run simulation for prediction; unable to create prediction success table.")
return
if log_to_file is not None and len(log_to_file) > 0:
logger.enable_file_logging(log_to_file)
## by default, compare predicted choice with observed choice
## this is not feasible for location choice model, where the
## alternative set is too large to be useful
if summarize_by is None:
summarize_by = "%s.%s" % (agents.dataset_name, choice_id_name)
summarize_dataset_name = VariableName(summarize_by).get_dataset_name()
if summarize_dataset_name == choices.dataset_name:
summary_id = choices.compute_variables(summarize_by)
chosen_choice_id = agents.get_attribute_by_index(choices.get_id_name()[0], agents_index)
predicted_choice_id = agents.get_attribute_by_index(predicted_choice_id_name, agents_index)
chosen_choice_index = choices.get_id_index(chosen_choice_id)
predicted_choice_index = choices.try_get_id_index(predicted_choice_id)
chosen_summary_id = summary_id[chosen_choice_index]
predicted_summary_id = summary_id[predicted_choice_index]
unique_summary_id = unique(summary_id)
elif summarize_dataset_name == agents.dataset_name:
chosen_summary_id = agents.compute_variables(summarize_by)[agents_index]
chosen_choice_id = agents.get_attribute(choice_id_name).copy()
predicted_choice_id = agents.get_attribute(predicted_choice_id_name)
agents.modify_attribute(name=choice_id_name, data=predicted_choice_id)
predicted_summary_id = agents.compute_variables(summarize_by)[agents_index]
agents.modify_attribute(name=choice_id_name, data=chosen_choice_id)
unique_summary_id = unique( concatenate((chosen_summary_id, predicted_summary_id)) )
else:
logger.log_error("summarize_by expression '%s' is specified for dataset %s, which is neither the choice_set '%s' nor the agent_set '%s'."
% (summarize_by, summarize_dataset_name, choices.dataset_name, agents.dataset_name))
return False
unique_nonneg_summary_id = unique_summary_id[unique_summary_id >= 0]
# observed on row, predicted on column
prediction_matrix = zeros( (unique_nonneg_summary_id.size, unique_nonneg_summary_id.size), dtype="int32" )
def _convert_array_to_tab_delimited_string(an_array):
from numpy import dtype
if an_array.dtype == dtype('f'):
return "\t".join(["%5.4f" % item for item in an_array])
return "\t".join([str(item) for item in an_array])
logger.log_status("Observed_id\tSuccess_rate\t%s" % \
_convert_array_to_tab_delimited_string(unique_nonneg_summary_id) )
i = 0
total_correct = 0
success_rate = zeros( unique_nonneg_summary_id.size, dtype="float32" )
for observed_id in unique_nonneg_summary_id:
predicted_id = predicted_summary_id[chosen_summary_id==observed_id]
prediction_matrix[i] = ndimage.sum(ones(predicted_id.size), labels=predicted_id, index=unique_nonneg_summary_id )
if prediction_matrix[i].sum() > 0:
if prediction_matrix[i].sum() > 0:
success_rate[i] = float(prediction_matrix[i, i]) / prediction_matrix[i].sum()
total_correct = total_correct + prediction_matrix[i, i]
else:
success_rate[i] = 0
logger.log_status("%s\t\t%5.4f\t\t%s" % (observed_id, success_rate[i],
_convert_array_to_tab_delimited_string(prediction_matrix[i]) ) )
i+=1
success_rate2 = zeros( i, dtype="float32" )
for j in range(i):
if prediction_matrix[j, :].sum() > 0:
success_rate2[j]=float(prediction_matrix[:,j].sum()) / prediction_matrix[j, :].sum()
else:
success_rate2[j]=0
logger.log_status("%s\t\t%s\t\t%s" % (' ', ' ',
_convert_array_to_tab_delimited_string( success_rate2 ) ))
logger.log_status("\nTotal success rate: %5.4f" % (total_correct/float(prediction_matrix.sum())))
logger.disable_file_logging(filename=log_to_file)
