def compute(self, dataset_pool): """ zone_ids = zone_set.get_attribute('zone_id') am_VMT = travel_data.get_attribute(self._am_VMT_attr) md_VMT = travel_data.get_attribute(self._md_VMT_attr) pm_ev_ni_VMT = travel_data.get_attribute(self._pm_ev_ni_VMT_attr) """ zone_ids = self.get_dataset().get_id_attribute() travel_data = dataset_pool.get_dataset('travel_data') from_zone_id = travel_data.get_attribute("from_zone_id") am_VMT_attr = travel_data.get_attribute(self._am_VMT_attr) md_VMT_attr = travel_data.get_attribute(self._md_VMT_attr) pm_ev_ni_VMT_attr = travel_data.get_attribute(self._pm_ev_ni_VMT_attr) non_missing_idx = logical_and(logical_and(am_VMT_attr <> self.missing_value, md_VMT_attr <> self.missing_value), pm_ev_ni_VMT_attr <> self.missing_value) results = array(ndimage_sum(am_VMT_attr[non_missing_idx], labels = from_zone_id[non_missing_idx], index=zone_ids)) + \ array(ndimage_sum(md_VMT_attr[non_missing_idx], labels = from_zone_id[non_missing_idx], index=zone_ids)) + \ array(ndimage_sum(pm_ev_ni_VMT_attr[non_missing_idx], labels = from_zone_id[non_missing_idx], index=zone_ids)) return results
def compute(self, dataset_pool): zone_set = self.get_dataset() travel_data = dataset_pool.get_dataset('travel_data') logsum_hbw_am_income = travel_data.get_attribute(self.logsum_hbw_am_income) trips_hbw_am_income = travel_data.get_attribute(self.trips_hbw_am_income) from_zone_id = travel_data.get_attribute('from_zone_id') zone_ids = zone_set.get_attribute('zone_id') numerator = array(ndimage_sum(travel_data.get_attribute(self.trips_hbw_am_income) * travel_data.get_attribute(self.logsum_hbw_am_income), labels = from_zone_id, index=zone_ids)) denominator = array(ndimage_sum(travel_data.get_attribute(self.trips_hbw_am_income), labels = from_zone_id, index=zone_ids), dtype=float32) # if there is a divide by zero then subsititute the values from the zone one below that one # if there are contigious places of zero division the values should propigate upon iteration no_trips_from_here = where(denominator == 0)[0] while no_trips_from_here.size != 0: substitute_locations = no_trips_from_here - 1 # a mapping, what zone the new data will come from if substitute_locations[0] < 0: substitute_locations[0] = 1 numerator[no_trips_from_here] = numerator[substitute_locations] denominator[no_trips_from_here] = denominator[substitute_locations] no_trips_from_here = where(denominator == 0)[0] return numerator / denominator
def get_average_omega(self, omega, probability, index, nsupply, demand): omega_prob = omega[:, newaxis]*probability omega_prob_sum_over_i = array(ndimage_sum(omega_prob, labels=index+1, index=arange(nsupply)+1)) prob_sum_over_i = array(ndimage_sum(probability, labels=index+1, index=arange(nsupply)+1)) average_omega = ma.filled(omega_prob_sum_over_i/ ma.masked_where(prob_sum_over_i==0, prob_sum_over_i), 0.0) return average_omega
def compute(self, dataset_pool): zone_set = self.get_dataset() travel_data = dataset_pool.get_dataset('travel_data') from_zone_id = travel_data.get_attribute('from_zone_id') zone_ids = zone_set.get_attribute('zone_id') time = travel_data.get_attribute(self.time_attribute_name) trips = travel_data.get_attribute(self.trips_attribute_name) non_missing_idx = where(logical_and(time <> self.missing_value, trips <> self.missing_value)) numerator = array(ndimage_sum(time[non_missing_idx] * trips[non_missing_idx], labels = from_zone_id[non_missing_idx], index=zone_ids)) denominator = array(ndimage_sum(trips[non_missing_idx], labels = from_zone_id[non_missing_idx], index=zone_ids), dtype=float32) # if there is a divide by zero then substitute the values from the zone below that one # if there are contiguous places of zero division the values should propagate upon iteration no_trips_from_here = where(denominator == 0)[0] while no_trips_from_here.size != 0: if no_trips_from_here.size == denominator.size: logger.log_warning("%s attribute of travel_data is all zeros; %s returns all zeros" % (self.trips_attribute_name, self.name() )) break substitute_locations = no_trips_from_here - 1 # a mapping, what zone the new data will come from if substitute_locations[0] < 0: substitute_locations[0] = 1 numerator[no_trips_from_here] = numerator[substitute_locations] denominator[no_trips_from_here] = denominator[substitute_locations] no_trips_from_here = where(denominator == 0)[0] return safe_array_divide(numerator, denominator)
def get_average_omega(self, omega, probability, index, nsupply, demand): omega_prob = omega[:, newaxis] * probability omega_prob_sum_over_i = array( ndimage_sum(omega_prob, labels=index + 1, index=arange(nsupply) + 1)) prob_sum_over_i = array( ndimage_sum(probability, labels=index + 1, index=arange(nsupply) + 1)) average_omega = ma.filled( omega_prob_sum_over_i / ma.masked_where(prob_sum_over_i == 0, prob_sum_over_i), 0.0) return average_omega
def compute(self, dataset_pool): zone_set = self.get_dataset() zone_ids = zone_set.get_attribute('zone_id') numerator = zeros(len(zone_ids)).astype(float32) for mode in self.numerator_modes: numerator += ndimage_sum(zone_set.get_attribute(mode), labels = zone_ids, index=zone_ids) denominator = zeros(len(zone_ids)).astype(float32) for mode in self.denominator_modes: denominator += ndimage_sum(zone_set.get_attribute(mode), labels = zone_ids, index=zone_ids) """if there is a divide by zero then replace with 1""" denominator[where(denominator == 0)] = 1 return numerator / denominator
def get_demand(self, index, probability, nsupply): flat_index = index.ravel() l = flat_index + 1 demand = array( ndimage_sum(probability.ravel(), labels=l, index=arange(nsupply) + 1)) return demand
def compute(self, dataset_pool): zone_set = self.get_dataset() travel_data = dataset_pool.get_dataset('travel_data') logsum_hbw_am_income = travel_data.get_attribute(self.logsum_hbw_am_income) from_zone_id = travel_data.get_attribute('from_zone_id') to_zone_id = travel_data.get_attribute('to_zone_id') td_attr_by_idx = travel_data.get_attribute_by_index zone_ids = zone_set.get_attribute('zone_id') nhouseholds = zone_set.get_attribute_by_id(self.number_of_households, from_zone_id) sums_by_from_zone = ndimage_sum(nhouseholds * exp(travel_data.get_attribute(self.logsum_hbw_am_income)), labels=to_zone_id, index=zone_ids) return array(sums_by_from_zone)
def compute(self, dataset_pool): zone_set = self.get_dataset() travel_data = dataset_pool.get_dataset('travel_data') from_zone_id = travel_data.get_attribute('from_zone_id') zone_ids = zone_set.get_attribute('zone_id') results = zeros(len(zone_ids)).astype(float32) for matrix in self.matrices: values = travel_data.get_attribute(matrix) non_missing_idx = where(values <> self.missing_value) results += array(ndimage_sum(values[non_missing_idx], labels = from_zone_id[non_missing_idx], index=zone_ids)) return results
def _simulate_submodel(self, submodel, location_set, agent_set, agents_index, ignore_agents_distribution=False): location_id_name = location_set.get_id_name()[0] subm_agents_index = agents_index[self.observations_mapping[submodel]] if self.submodel_string is not None: all_agents_in_subm = where(agent_set[self.submodel_string]==submodel)[0] else: all_agents_in_subm = arange(agent_set.size()) if subm_agents_index.size <= 0: return array([], dtype='int32') #unplace agents agent_set.set_values_of_one_attribute(location_id_name, resize(array([-1]), subm_agents_index.size), subm_agents_index) if not ignore_agents_distribution: agent_distr_in_loc = array(ndimage_sum(ones(all_agents_in_subm.size), labels=agent_set[location_id_name][all_agents_in_subm], index=location_set.get_id_attribute())) else: agent_distr_in_loc = ones(location_set.size(), dtype="int32") location_ind = ones(location_set.size(), dtype='bool') if self.filter is not None: submodel_filter = re.sub('SUBMODEL', str(submodel), self.filter) filter_values = location_set.compute_variables([submodel_filter], dataset_pool=self.dataset_pool) location_ind = logical_and(location_ind, filter_values > 0) if self.weights is not None: submodel_weights = re.sub('SUBMODEL', str(submodel), self.weights) weight_values = location_set.compute_variables([submodel_weights], dataset_pool=self.dataset_pool) location_ind = logical_and(location_ind, weight_values > 0) location_index = where(location_ind)[0] if location_index.size <= 0: logger.log_status("No locations available. Nothing to be done.") return array(subm_agents_index.size*[-1], dtype="int32") logger.log_status("Submodel %s: %s %s(s) are scaled into %s %s(s)." % (submodel, subm_agents_index.size, agent_set.get_dataset_name(), location_index.size, location_set.get_dataset_name())) distr = agent_distr_in_loc[location_index] if self.weights is not None: distr = distr * weight_values[location_index] if ma.allclose(distr.sum(), 0): uniform_prob = 1.0/distr.size distr = resize(array([uniform_prob], dtype='float64'), distr.size) logger.log_warning("Probabilities in scaling model for submodel " + str(submodel) + " sum to 0.0. Substituting uniform distribution!") distr = distr/float(distr.sum()) random_sample = probsample_replace(location_set.get_id_attribute()[location_index], size=subm_agents_index.size, prob_array=distr) #modify agents locations agent_set.set_values_of_one_attribute(location_id_name, random_sample, subm_agents_index) return random_sample
def _do_run(self, location_set, agent_set, agents_index, data_objects=None, resources=None): location_id_name = location_set.get_id_name()[0] jobsubset = DatasetSubset(agent_set, agents_index) if jobsubset.size() <= 0: return array([], dtype='int32') #unplace jobs agent_set.set_values_of_one_attribute(location_id_name, resize(array([-1.0]), jobsubset.size()), agents_index) sector_ids = jobsubset.get_attribute("sector_id") sectors = unique(sector_ids) counts = ndimage_sum(ones((jobsubset.size(),)), labels=sector_ids.astype('int32'), index=sectors.astype('int32')) if sectors.size <=1 : counts = array([counts]) variables = map(lambda x: "number_of_jobs_of_sector_"+str(int(x)), sectors) compute_variables = map(lambda var: self.variable_package + "." + location_set.get_dataset_name()+ "." + var, variables) if data_objects is not None: self.dataset_pool.add_datasets_if_not_included(data_objects) self.dataset_pool.add_datasets_if_not_included({agent_set.get_dataset_name():agent_set}) location_set.compute_variables(compute_variables, dataset_pool=self.dataset_pool) if self.filter is None: location_index = arange(location_set.size()) else: filter_values = location_set.compute_variables([self.filter], dataset_pool=self.dataset_pool) location_index = where(filter_values > 0)[0] if location_index.size <= 0: logger.log_status("No locations available. Nothing to be done.") return array([]) location_subset = DatasetSubset(location_set, location_index) i=0 for sector in sectors: distr = location_subset.get_attribute(variables[i]) if ma.allclose(distr.sum(), 0): uniform_prob = 1.0/distr.size distr = resize(array([uniform_prob], dtype='float64'), distr.size) logger.log_warning("Probabilities in scaling model for sector " + str(sector) + " sum to 0.0. Substituting uniform distribution!") # random_sample = sample(location_set.get_attribute("grid_id"), k=int(counts[i]), \ # probabilities = distr) distr = distr/float(distr.sum()) random_sample = probsample_replace(location_subset.get_id_attribute(), size=int(counts[i]), prob_array=distr) idx = where(sector_ids == sector)[0] #modify job locations agent_set.set_values_of_one_attribute(location_id_name, random_sample, agents_index[idx]) i+=1 return agent_set.get_attribute_by_index(location_id_name, agents_index)
def get_demand(self, index, probability, nsupply): flat_index = index.ravel() l = flat_index + 1 demand = array(ndimage_sum(probability.ravel(), labels=l, index=arange(nsupply)+1)) return demand
def run(self, probability, resources=None): """ Compute choices according to given probability -- Constrain Location Choice procedure. 'probability' is a 2D numpy array (nobservation x nequations). The returned value is a 1D array of choice indices [0, nequations-1] of the length nobservations). The argument 'resources' must contain an entry 'capacity'. It is 1D array whose number of elements corresponds to the number of choices. Optional entry 'index' (1D or 2D array) gives indices of the choices. """ if probability.ndim < 2: raise StandardError, "Argument 'probability' must be a 2D numpy array." resources.check_obligatory_keys(["capacity"]) supply = resources["capacity"] if not isinstance(supply, ndarray): supply = array(supply) nsupply = supply.size # logger.log_status('Supply.shape:',supply.shape) # logger.log_status('supply.sum:', supply.sum()) max_iter = resources.get("max_iterations", None) if max_iter == None: max_iter = 100 # default index = resources.get("index", None) if index == None: index = arange(nsupply) # logger.log_status('index.shape:',index.shape) neqs = probability.shape[1] nobs = probability.shape[0] if supply.sum < nobs: raise StandardError, "Aggregate Supply Must be Greater than Aggregate Demand." if index.ndim <= 1: index = repeat(reshape(index, (1,index.shape[0])), nobs) resources.merge({"index":index}) # logger.log_status('index.shape:',index.shape) flat_index = index.ravel() unique_index = unique(flat_index) # logger.log_status('flat_index.shape:',flat_index.shape) # logger.log_status('unique_index.shape',unique_index.shape) # logger.log_status(unique_index) l = flat_index + 1 demand = array(ndimage_sum(probability.ravel(), labels=l, index=arange(nsupply)+1)) # logger.log_status('demand.shape:',demand.shape) # logger.log_status('demand.sum:', demand.sum()) # logger.log_status('probability.sum:',probability.sum()) #initial calculations sdratio = ma.filled(supply/ma.masked_where(demand==0, demand),1.0) # logger.log_status('sdratio.shape:',sdratio.shape) constrained_locations = where(sdratio<1,1,0) unconstrained_locations = 1-constrained_locations # Compute the iteration zero omegas sdratio_matrix = sdratio[index] constrained_locations_matrix = constrained_locations[index] unconstrained_locations_matrix = unconstrained_locations[index] prob_sum = 1-(probability*constrained_locations_matrix).sum(axis=1) omega = (1-(probability*constrained_locations_matrix*sdratio_matrix).sum(axis=1))/ \ ma.masked_where(prob_sum ==0, prob_sum) pi = sdratio_matrix / ma.resize(omega, (nobs,1)) * constrained_locations_matrix + unconstrained_locations_matrix average_omega = ma.filled((ma.resize(omega,(nobs,1))*probability).sum(axis=0)/\ ma.masked_where(demand[index]==0, demand[index]),0.0) number_constrained_locations=zeros((max_iter,)) # Iterative Constrained Location Procedure for i in range(max_iter): logger.log_status('Iteration ',i+1, 'Average Omega:',average_omega[0:4]) # Recompute the constrained locations using iteration zero value of Omega constrained_locations_matrix = where(supply[index]<(average_omega*demand[index]),1,0) unconstrained_locations_matrix = 1-constrained_locations_matrix # Update values of Omega using new Constrained Locations prob_sum = 1-(probability*constrained_locations_matrix).sum(axis=1) omega = (1-(probability*constrained_locations_matrix*sdratio_matrix).sum(axis=1))/\ ma.masked_where(prob_sum ==0, prob_sum) # pi = sdratio_matrix / ma.resize(omega, (nobs,1)) * constrained_locations_matrix + unconstrained_locations_matrix # logger.log_status('sdratio_matrix',sdratio_matrix.shape) # logger.log_status('constrained_locations_matrix',constrained_locations_matrix.shape) # logger.log_status('omega',omega.shape) # logger.log_status('unconstrained_locations_matrix',unconstrained_locations_matrix.shape) # pi_ta = (sdratio_matrix*constrained_locations_matrix) # logger.log_status('pi+ta',pi_ta.shape) # pi_tb = ma.resize(omega,(nobs,neqs))*unconstrained_locations_matrix # logger.log_status('pi_tb',pi_tb.shape) pi_t = (sdratio_matrix*constrained_locations_matrix)+ma.resize(omega,(nobs,neqs))*unconstrained_locations_matrix # logger.log_status('pi_tilde:',pi_t.shape) # Update the values of average Omegas per alternative average_omega = ma.filled((ma.resize(omega,(nobs,1))*probability).sum(axis=0)/ ma.masked_where(demand[index]==0, demand[index]),0.0) number_constrained_locations[i]= constrained_locations_matrix.sum() # Test for Convergence and if Reached, Exit if i > 0: if number_constrained_locations[i] == number_constrained_locations[i-1]: break # update probabilities # new_probability = ma.filled(probability*ma.resize(omega,(nobs,1))*pi,0.0) new_probability = ma.filled(probability*pi_t,0.0) choices = lottery_choices().run(new_probability, resources) return choices
def run(self, probability, resources=None): """ Compute choices according to given probability -- Constrain Location Choice procedure. 'probability' is a 2D numpy array (nobservation x nequations). The returned value is a 1D array of choice indices [0, nequations-1] of the length nobservations). The argument 'resources' must contain an entry 'capacity'. It is 1D array whose number of elements corresponds to the number of choices. Optional entry 'index' (1D or 2D array) gives indices of the choices. """ if probability.ndim < 2: raise StandardError, "Argument 'probability' must be a 2D numpy array." resources.check_obligatory_keys(["capacity"]) supply = resources["capacity"] if not isinstance(supply, ndarray): supply = array(supply) nsupply = supply.size max_iter = resources.get("max_iterations", None) if max_iter == None: max_iter = 100 # default index = resources.get("index", None) if index == None: index = arange(nsupply) neqs = probability.shape[1] nobs = probability.shape[0] if index.ndim <= 1: index = repeat(reshape(index, (1,index.shape[0])), nobs) resources.merge({"index":index}) flat_index = index.ravel() unique_index = unique(flat_index) l = flat_index + 1 demand = array(ndimage_sum(probability.ravel(), labels=l, index=arange(nsupply)+1)) #initial calculations sdratio = ma.filled(supply/ma.masked_where(demand==0, demand),2.0) constrained_locations = logical_and(sdratio<1,demand-supply>0.1).astype("int8") unconstrained_locations = 1-constrained_locations excess_demand = (demand-supply)*constrained_locations global_excess_demand = excess_demand.sum() # Compute the iteration zero omegas sdratio_matrix = sdratio[index] constrained_locations_matrix = constrained_locations[index] # Would like to include following print statements in debug printing # logger.log_status('Total demand:',demand.sum()) # logger.log_status('Total supply:',supply.sum()) logger.log_status('Global excess demand:',global_excess_demand) # logger.log_status('Constrained locations:',constrained_locations.sum()) unconstrained_locations_matrix = unconstrained_locations[index] prob_sum = 1-(probability*constrained_locations_matrix).sum(axis=1) # The recoding of prob_sum and omega are to handle extreme values of omega and zero divide problems # A complete solution involves stratifying the choice set in the initialization to ensure that # there are always a mixture of constrained and unconstrained alternatives in each choice set. prob_sum = where(prob_sum==0,-1,prob_sum) omega = (1-(probability*constrained_locations_matrix*sdratio_matrix).sum(axis=1))/prob_sum omega = where(omega>5,5,omega) omega = where(omega<.5,5,omega) omega = where(prob_sum<0,5,omega) # Debug print statements # logger.log_status('Minimum omega',minimum(omega)) # logger.log_status('Maximum omega',maximum(omega)) # logger.log_status('Median omega',median(omega)) # logger.log_status('Omega < 0',(where(omega<0,1,0)).sum()) # logger.log_status('Omega < 1',(where(omega<1,1,0)).sum()) # logger.log_status('Omega > 30',(where(omega>30,1,0)).sum()) # logger.log_status('Omega > 100',(where(omega>100,1,0)).sum()) # logger.log_status('Omega histogram:',histogram(omega,0,30,30)) # logger.log_status('Excess demand max:',maximum(excess_demand)) # logger.log_status('Excess demand 0-1000:',histogram(excess_demand,0,1000,20)) # logger.log_status('Excess demand 0-10:',histogram(excess_demand,0,10,20)) pi = sdratio_matrix / ma.resize(omega, (nobs,1)) * constrained_locations_matrix + unconstrained_locations_matrix omega_prob = ma.filled(ma.resize(omega,(nobs,1))*probability,0.0) average_omega_nom = array(ndimage_sum(omega_prob, labels=index+1, index=arange(nsupply)+1)) average_omega = ma.filled(average_omega_nom/ ma.masked_where(demand==0, demand), 0.0) # logger.log_status('Total demand:',new_demand.sum()) # logger.log_status('Excess demand:',excess_demand) number_constrained_locations=zeros((max_iter,)) # Iterative Constrained Location Procedure for i in range(max_iter): logger.log_status() logger.log_status('Constrained location choice iteration ',i+1) # Recompute the constrained locations using preceding iteration value of Omega constrained_locations = where((average_omega*demand-supply>0.1),1,0) unconstrained_locations = 1-constrained_locations constrained_locations_matrix = constrained_locations[index] unconstrained_locations_matrix = unconstrained_locations[index] # logger.log_status('supply.shape,average_omega.shape,demand.shape',supply.shape,average_omega.shape,demand.shape) # logger.log_status('constrained_locations_matrix',constrained_locations_matrix) # logger.log_status('constrained_locations_matrix.shape',constrained_locations_matrix.shape) # logger.log_status('unconstrained_locations_matrix',unconstrained_locations_matrix) # Update values of Omega using new Constrained Locations prob_sum = 1-(probability*constrained_locations_matrix).sum(axis=1) prob_sum = where(prob_sum==0,-1,prob_sum) omega = (1-(probability*constrained_locations_matrix*sdratio_matrix).sum(axis=1))/prob_sum omega = where(omega>5,5,omega) omega = where(omega<.5,5,omega) omega = where(prob_sum<0,5,omega) pi = sdratio_matrix / ma.resize(omega, (nobs,1)) * constrained_locations_matrix + unconstrained_locations_matrix # Update the values of average Omegas per alternative omega_prob = ma.filled(ma.resize(omega,(nobs,1)), 1.0)*probability average_omega_num = array(ndimage_sum(omega_prob, labels=index+1, index=arange(nsupply)+1)) average_omega = ma.filled(average_omega_num/ ma.masked_where(demand==0, demand), 0.0) number_constrained_locations[i] = constrained_locations.sum() new_probability = ma.filled(probability*ma.resize(omega,(nobs,1))*pi,0.0) new_demand = array(ndimage_sum(new_probability.ravel(), labels=l, index=arange(nsupply)+1)) excess_demand = (new_demand-supply)*constrained_locations global_excess_demand = excess_demand.sum() # logger.log_status('Total demand:',new_demand.sum()) logger.log_status('Global excess demand:',global_excess_demand) # logger.log_status('Constrained locations:', number_constrained_locations[i]) # logger.log_status('Minimum omega',minimum(omega)) # logger.log_status('Maximum omega',maximum(omega)) # logger.log_status('Median omega',median(omega)) # logger.log_status('Omega < 0',(where(omega<0,1,0)).sum()) # logger.log_status('Omega < 1',(where(omega<1,1,0)).sum()) # logger.log_status('Omega > 30',(where(omega>30,1,0)).sum()) # logger.log_status('Omega > 100',(where(omega>100,1,0)).sum()) # logger.log_status('Omega histogram:',histogram(omega,0,30,30)) # logger.log_status('Excess demand max:',maximum(excess_demand)) # logger.log_status('Excess demand 0-5:',histogram(excess_demand,0,5,20)) # logger.log_status('Excess demand 0-1:',histogram(excess_demand,0,1,20)) # Test for Convergence and if Reached, Exit if i > 0: if number_constrained_locations[i] == number_constrained_locations[i-1]: logger.log_status() logger.log_status('Constrained choices converged.') break # update probabilities new_probability = ma.filled(probability*ma.resize(omega,(nobs,1))*pi,0.0) choices = lottery_choices().run(new_probability, resources) return choices
def get_choice_histogram(self, units_to_occupy, choices, nchoices): """Return a histogram of agent choices, where each agents occupy number of units given in 'units_to_occupy'. 'choices' are the agent choices of a location (as an index). 'nchoices' is a number of unique values for possible choices. """ return array(ndimage_sum(units_to_occupy, labels=choices+1, index=arange(nchoices)+1))
def _do_run(self, location_set, agent_set, agents_index, data_objects=None, resources=None): location_id_name = location_set.get_id_name()[0] jobsubset = DatasetSubset(agent_set, agents_index) if jobsubset.size() <= 0: return array([], dtype='int32') #unplace jobs agent_set.set_values_of_one_attribute( location_id_name, resize(array([-1.0]), jobsubset.size()), agents_index) sector_ids = jobsubset.get_attribute("sector_id") sectors = unique(sector_ids) counts = ndimage_sum(ones((jobsubset.size(), )), labels=sector_ids.astype('int32'), index=sectors.astype('int32')) if sectors.size <= 1: counts = array([counts]) variables = map(lambda x: "number_of_jobs_of_sector_" + str(int(x)), sectors) compute_variables = map( lambda var: self.variable_package + "." + location_set. get_dataset_name() + "." + var, variables) if data_objects is not None: self.dataset_pool.add_datasets_if_not_included(data_objects) self.dataset_pool.add_datasets_if_not_included( {agent_set.get_dataset_name(): agent_set}) location_set.compute_variables(compute_variables, dataset_pool=self.dataset_pool) if self.filter is None: location_index = arange(location_set.size()) else: filter_values = location_set.compute_variables( [self.filter], dataset_pool=self.dataset_pool) location_index = where(filter_values > 0)[0] if location_index.size <= 0: logger.log_status("No locations available. Nothing to be done.") return array([]) location_subset = DatasetSubset(location_set, location_index) i = 0 for sector in sectors: distr = location_subset.get_attribute(variables[i]) if ma.allclose(distr.sum(), 0): uniform_prob = 1.0 / distr.size distr = resize(array([uniform_prob], dtype='float64'), distr.size) logger.log_warning( "Probabilities in scaling model for sector " + str(sector) + " sum to 0.0. Substituting uniform distribution!") # random_sample = sample(location_set.get_attribute("grid_id"), k=int(counts[i]), \ # probabilities = distr) distr = distr / float(distr.sum()) random_sample = probsample_replace( location_subset.get_id_attribute(), size=int(counts[i]), prob_array=distr) idx = where(sector_ids == sector)[0] #modify job locations agent_set.set_values_of_one_attribute(location_id_name, random_sample, agents_index[idx]) i += 1 return agent_set.get_attribute_by_index(location_id_name, agents_index)
def _add(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool ): fit_index = self.get_fit_agents_index(agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if this_refinement.agent_expression is not None and len(this_refinement.agent_expression) > 0: agents_index = where(agent_dataset.compute_variables(this_refinement.agent_expression, dataset_pool=dataset_pool)>0)[0] else: agents_index = arange(agent_dataset.size()) movers_index = array([],dtype="int32") ar_pool = array(agents_pool) fitted_agents_pool = ar_pool[in1d(ar_pool, agents_index)] amount_from_agents_pool = min( amount, fitted_agents_pool.size ) prob_string = self.probability_attributes.get(agent_dataset.get_dataset_name(),None) if prob_string is not None: probs_values = (agent_dataset.compute_variables([prob_string], dataset_pool=dataset_pool)).astype('int32') uprobs_values = unique(probs_values[fit_index]) if uprobs_values.size > 0: probs_existing = array(ndimage_sum(ones(fit_index.size), labels=probs_values[fit_index], index=uprobs_values)) if amount_from_agents_pool > 0: if prob_string is not None and uprobs_values.size > 0: prob_pool_values = probs_values[fitted_agents_pool] probs_pool=zeros(prob_pool_values.size) for i in range(uprobs_values.size): probpoolidx = where(prob_pool_values == uprobs_values[i])[0] if probpoolidx.size == 0: continue probs_pool[probpoolidx]=probs_existing[i]/float(probpoolidx.size) probs_pool[probs_pool<=0] = (probs_existing.min()/10.0)/float((probs_pool<=0).sum()) else: probs_pool=ones(fitted_agents_pool.size) agents_index_from_agents_pool = probsample_noreplace( fitted_agents_pool, amount_from_agents_pool, prob_array=probs_pool ) [ agents_pool.remove(i) for i in agents_index_from_agents_pool ] if fit_index.size == 0: ##cannot find agents to copy their location or clone them, place agents in agents_pool if amount > amount_from_agents_pool: logger.log_warning("Refinement requests to add %i agents, but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \ "add %i agents instead" % (amount, amount_from_agents_pool, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression]).strip(' and '), amount_from_agents_pool,) ) amount = amount_from_agents_pool # sample from all suitable locations is_suitable_location = location_dataset.compute_variables( this_refinement.location_expression, dataset_pool=dataset_pool ) location_id_for_agents_pool = sample_replace( location_dataset.get_id_attribute()[is_suitable_location], amount_from_agents_pool ) else: #sample from locations of suitable agents agents_index_for_location = sample_replace( fit_index, amount_from_agents_pool) location_id_for_agents_pool = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[agents_index_for_location] movers_index = concatenate( (movers_index, agents_index_for_location) ) elif fit_index.size == 0: ## no agents in agents_pool and no agents to clone either, --> fail logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \ ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') ) return if amount > amount_from_agents_pool: agents_index_to_clone = sample_replace( fit_index, amount - amount_from_agents_pool) movers_index = concatenate( (movers_index, agents_index_to_clone) ) if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0: movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id ) # see previous comment about histogram function num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ ( location_dataset.dataset_name, agent_dataset.dataset_name ), dataset_pool=dataset_pool) expand_factor = safe_array_divide( (num_of_agents_by_location + num_of_movers_by_location ).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 ) new_values = round_( expand_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) ) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values ) self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index) if amount_from_agents_pool > 0: agent_dataset.modify_attribute( 'building_id', -1 * ones( agents_index_from_agents_pool.size, dtype='int32' ), agents_index_from_agents_pool ) agent_dataset.modify_attribute( location_dataset.get_id_name()[0], location_id_for_agents_pool, agents_index_from_agents_pool ) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_from_agents_pool) self.processed_locations['add'] = concatenate((self.processed_locations.get('add', array([])), unique(location_dataset[self.subarea_id_name][location_dataset.get_id_index(location_id_for_agents_pool)]))) if amount > amount_from_agents_pool: new_agents_index = agent_dataset.duplicate_rows(agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=new_agents_index) if location_dataset.get_dataset_name() <> 'building': agent_dataset.modify_attribute( 'building_id', -1 * ones( new_agents_index.size, dtype='int32' ), new_agents_index ) self.processed_locations['add'] = concatenate((self.processed_locations.get('add', array([])), unique(agent_dataset[self.subarea_id_name][new_agents_index])))
def run(self, probability, resources=None): """ Compute choices according to given probability -- Constrain Location Choice procedure. 'probability' is a 2D numpy array (nobservation x nequations). The returned value is a 1D array of choice indices [0, nequations-1] of the length nobservations). The argument 'resources' must contain an entry 'capacity'. It is 1D array whose number of elements corresponds to the number of choices. Optional entry 'index' (1D or 2D array) gives indices of the choices. """ if probability.ndim < 2: raise StandardError, "Argument 'probability' must be a 2D numpy array." resources.check_obligatory_keys(["capacity"]) supply = resources["capacity"] if not isinstance(supply, ndarray): supply = array(supply) nsupply = supply.size max_iter = resources.get("max_iterations", None) if max_iter == None: max_iter = 100 # default index = resources.get("index", None) if index == None: index = arange(nsupply) neqs = probability.shape[1] nobs = probability.shape[0] if index.ndim <= 1: index = repeat(reshape(index, (1, index.shape[0])), nobs) resources.merge({"index": index}) flat_index = index.ravel() unique_index = unique(flat_index) l = flat_index + 1 demand = array( ndimage_sum(probability.ravel(), labels=l, index=arange(nsupply) + 1)) #initial calculations sdratio = ma.filled(supply / ma.masked_where(demand == 0, demand), 2.0) constrained_locations = logical_and( sdratio < 1, demand - supply > 0.1).astype("int8") unconstrained_locations = 1 - constrained_locations excess_demand = (demand - supply) * constrained_locations global_excess_demand = excess_demand.sum() # Compute the iteration zero omegas sdratio_matrix = sdratio[index] constrained_locations_matrix = constrained_locations[index] # Would like to include following print statements in debug printing # logger.log_status('Total demand:',demand.sum()) # logger.log_status('Total supply:',supply.sum()) logger.log_status('Global excess demand:', global_excess_demand) # logger.log_status('Constrained locations:',constrained_locations.sum()) unconstrained_locations_matrix = unconstrained_locations[index] prob_sum = 1 - (probability * constrained_locations_matrix).sum(axis=1) # The recoding of prob_sum and omega are to handle extreme values of omega and zero divide problems # A complete solution involves stratifying the choice set in the initialization to ensure that # there are always a mixture of constrained and unconstrained alternatives in each choice set. prob_sum = where(prob_sum == 0, -1, prob_sum) omega = (1 - (probability * constrained_locations_matrix * sdratio_matrix).sum(axis=1)) / prob_sum omega = where(omega > 5, 5, omega) omega = where(omega < .5, 5, omega) omega = where(prob_sum < 0, 5, omega) # Debug print statements # logger.log_status('Minimum omega',minimum(omega)) # logger.log_status('Maximum omega',maximum(omega)) # logger.log_status('Median omega',median(omega)) # logger.log_status('Omega < 0',(where(omega<0,1,0)).sum()) # logger.log_status('Omega < 1',(where(omega<1,1,0)).sum()) # logger.log_status('Omega > 30',(where(omega>30,1,0)).sum()) # logger.log_status('Omega > 100',(where(omega>100,1,0)).sum()) # logger.log_status('Omega histogram:',histogram(omega,0,30,30)) # logger.log_status('Excess demand max:',maximum(excess_demand)) # logger.log_status('Excess demand 0-1000:',histogram(excess_demand,0,1000,20)) # logger.log_status('Excess demand 0-10:',histogram(excess_demand,0,10,20)) pi = sdratio_matrix / ma.resize(omega, ( nobs, 1)) * constrained_locations_matrix + unconstrained_locations_matrix omega_prob = ma.filled(ma.resize(omega, (nobs, 1)) * probability, 0.0) average_omega_nom = array( ndimage_sum(omega_prob, labels=index + 1, index=arange(nsupply) + 1)) average_omega = ma.filled( average_omega_nom / ma.masked_where(demand == 0, demand), 0.0) # logger.log_status('Total demand:',new_demand.sum()) # logger.log_status('Excess demand:',excess_demand) number_constrained_locations = zeros((max_iter, )) # Iterative Constrained Location Procedure for i in range(max_iter): logger.log_status() logger.log_status('Constrained location choice iteration ', i + 1) # Recompute the constrained locations using preceding iteration value of Omega constrained_locations = where( (average_omega * demand - supply > 0.1), 1, 0) unconstrained_locations = 1 - constrained_locations constrained_locations_matrix = constrained_locations[index] unconstrained_locations_matrix = unconstrained_locations[index] # logger.log_status('supply.shape,average_omega.shape,demand.shape',supply.shape,average_omega.shape,demand.shape) # logger.log_status('constrained_locations_matrix',constrained_locations_matrix) # logger.log_status('constrained_locations_matrix.shape',constrained_locations_matrix.shape) # logger.log_status('unconstrained_locations_matrix',unconstrained_locations_matrix) # Update values of Omega using new Constrained Locations prob_sum = 1 - (probability * constrained_locations_matrix).sum(axis=1) prob_sum = where(prob_sum == 0, -1, prob_sum) omega = (1 - (probability * constrained_locations_matrix * sdratio_matrix).sum(axis=1)) / prob_sum omega = where(omega > 5, 5, omega) omega = where(omega < .5, 5, omega) omega = where(prob_sum < 0, 5, omega) pi = sdratio_matrix / ma.resize( omega, (nobs, 1) ) * constrained_locations_matrix + unconstrained_locations_matrix # Update the values of average Omegas per alternative omega_prob = ma.filled(ma.resize(omega, (nobs, 1)), 1.0) * probability average_omega_num = array( ndimage_sum(omega_prob, labels=index + 1, index=arange(nsupply) + 1)) average_omega = ma.filled( average_omega_num / ma.masked_where(demand == 0, demand), 0.0) number_constrained_locations[i] = constrained_locations.sum() new_probability = ma.filled( probability * ma.resize(omega, (nobs, 1)) * pi, 0.0) new_demand = array( ndimage_sum(new_probability.ravel(), labels=l, index=arange(nsupply) + 1)) excess_demand = (new_demand - supply) * constrained_locations global_excess_demand = excess_demand.sum() # logger.log_status('Total demand:',new_demand.sum()) logger.log_status('Global excess demand:', global_excess_demand) # logger.log_status('Constrained locations:', number_constrained_locations[i]) # logger.log_status('Minimum omega',minimum(omega)) # logger.log_status('Maximum omega',maximum(omega)) # logger.log_status('Median omega',median(omega)) # logger.log_status('Omega < 0',(where(omega<0,1,0)).sum()) # logger.log_status('Omega < 1',(where(omega<1,1,0)).sum()) # logger.log_status('Omega > 30',(where(omega>30,1,0)).sum()) # logger.log_status('Omega > 100',(where(omega>100,1,0)).sum()) # logger.log_status('Omega histogram:',histogram(omega,0,30,30)) # logger.log_status('Excess demand max:',maximum(excess_demand)) # logger.log_status('Excess demand 0-5:',histogram(excess_demand,0,5,20)) # logger.log_status('Excess demand 0-1:',histogram(excess_demand,0,1,20)) # Test for Convergence and if Reached, Exit if i > 0: if number_constrained_locations[ i] == number_constrained_locations[i - 1]: logger.log_status() logger.log_status('Constrained choices converged.') break # update probabilities new_probability = ma.filled( probability * ma.resize(omega, (nobs, 1)) * pi, 0.0) choices = lottery_choices().run(new_probability, resources) return choices
def _do_run_for_this_year(self, job_set): building_type = job_set.get_attribute("building_type") sectors = unique(self.control_totals_for_this_year.get_attribute("sector_id")) self._compute_sector_variables(sectors, job_set) for sector in sectors: isector = where(self.control_totals_for_this_year.get_attribute("sector_id") == sector)[0] total_hb_jobs = self.control_totals_for_this_year.get_attribute("total_home_based_employment")[isector] total_nhb_jobs = self.control_totals_for_this_year.get_attribute("total_non_home_based_employment")[isector] is_in_sector_hb = job_set.get_attribute("is_in_employment_sector_%s_home_based" % sector) is_in_sector_nhb = job_set.get_attribute("is_in_employment_sector_%s_non_home_based" % sector) diff_hb = int(total_hb_jobs - is_in_sector_hb.astype(int8).sum()) diff_nhb = int(total_nhb_jobs - is_in_sector_nhb.astype(int8).sum()) if diff_hb < 0: # home based jobs to be removed w = where(is_in_sector_hb == 1)[0] sample_array, non_placed, size_non_placed = \ get_array_without_non_placed_agents(job_set, w, -1*diff_hb, self.location_id_name) self.remove_jobs = concatenate((self.remove_jobs, non_placed, sample_noreplace(sample_array, max(0,abs(diff_hb)-size_non_placed)))) if diff_nhb < 0: # non home based jobs to be removed w = where(is_in_sector_nhb == 1)[0] sample_array, non_placed, size_non_placed = \ get_array_without_non_placed_agents(job_set, w, -1*diff_nhb, self.location_id_name) self.remove_jobs = concatenate((self.remove_jobs, non_placed, sample_noreplace(sample_array, max(0,abs(diff_nhb)-size_non_placed)))) if diff_hb > 0: # home based jobs to be created self.new_jobs[self.location_id_name] = concatenate((self.new_jobs[self.location_id_name], zeros((diff_hb,), dtype=self.new_jobs[self.location_id_name].dtype.type))) self.new_jobs["sector_id"] = concatenate((self.new_jobs["sector_id"], (resize(array([sector], dtype=self.new_jobs["sector_id"].dtype.type), diff_hb)))) if 1 in is_in_sector_hb: building_type_distribution = array(ndimage_sum(is_in_sector_hb, labels=building_type, index=self.available_building_types)) elif 1 in job_set.get_attribute("is_home_based_job"): # take the building type distribution from the whole region building_type_distribution = array(ndimage_sum( job_set.get_attribute("is_home_based_job"), labels=building_type, index=self.available_building_types)) else: # there are no home-based jobs in the region, take uniform distribution building_type_distribution = ones(self.available_building_types.size) building_type_distribution = building_type_distribution/building_type_distribution.sum() sampled_building_types = probsample_replace( self.available_building_types, diff_hb, building_type_distribution/ float(building_type_distribution.sum())) self.new_jobs["building_type"] = concatenate((self.new_jobs["building_type"], sampled_building_types.astype(self.new_jobs["building_type"].dtype.type))) new_max_id = self.max_id + diff_hb self.new_jobs[self.job_id_name] = concatenate((self.new_jobs[self.job_id_name], arange(self.max_id+1, new_max_id+1))) self.max_id = new_max_id if diff_nhb > 0: # non home based jobs to be created self.new_jobs[self.location_id_name]=concatenate((self.new_jobs[self.location_id_name], zeros((diff_nhb,), dtype=self.new_jobs[self.location_id_name].dtype.type))) self.new_jobs["sector_id"]=concatenate((self.new_jobs["sector_id"], (resize(array([sector], dtype=self.new_jobs["sector_id"].dtype.type), diff_nhb)))) if 1 in is_in_sector_nhb: building_type_distribution = array(ndimage_sum(is_in_sector_nhb, labels=building_type, index=self.available_building_types)) elif 1 in job_set.get_attribute("is_non_home_based_job"): # take the building type distribution from the whole region building_type_distribution = array(ndimage_sum( job_set.get_attribute("is_non_home_based_job"), labels=building_type, index=self.available_building_types)) else: # there are no non-home-based jobs in the region, take uniform distribution building_type_distribution = ones(self.available_building_types.size) building_type_distribution = building_type_distribution/building_type_distribution.sum() sampled_building_types = probsample_replace( self.available_building_types, diff_nhb, building_type_distribution/ float(building_type_distribution.sum())) self.new_jobs["building_type"] = concatenate((self.new_jobs["building_type"], sampled_building_types.astype(self.new_jobs["building_type"].dtype.type))) new_max_id = self.max_id+diff_nhb self.new_jobs[self.job_id_name]=concatenate((self.new_jobs[self.job_id_name], arange(self.max_id+1, new_max_id+1))) self.max_id = new_max_id
def get_choice_histogram(self, units_to_occupy, choices, nchoices): """Counts the number of agents that decided for each choice. """ return array(ndimage_sum(ones((choices.size,)), labels=choices+1, index=arange(nchoices)+1))
def run(self, job_dataset, dataset_pool, out_storage=None, jobs_table="jobs"): """ Algorithm: 1. For all non_home_based jobs that have parcel_id assigned but no building_id, try to choose a building from all buildings in that parcel. Draw the building with probabilities given by the sector-building_type distribution. The job sizes are fitted into the available space (the attribute job.sqft is updated). 2. For all non_home_based jobs for which no building was found in step 1, check if the parcel has residential buildings. In such a case, re-assign the jobs to be home-based. Otherwise, if sum of non_residential_sqft over the involved buildings is 0, for all jobs that have impute_building_sqft_flag=True draw a building using the sector-building_type distribution and impute the corresponding sqft to the non_residential_sqft of that building. 3. For all home_based jobs that have parcel_id assigned but no building_id, try to choose a building from all buildings in that parcel. The capacity of a single-family building is determined from sizes of the households living there (for each household the minimum of number of members and 2 is taken). For multi-family buildings the capacity is 50. 4. Assign a building type to jobs that have missing building type. It is sampled from the regional-wide distribution of home based and non-home based jobs. 5. Update the table 'building_sqft_per_job' using the updated job.sqft. 'in_storage' should contain the jobs table and the zone_averages_table. The 'dataset_pool_storage' should contain all other tables needed (buildings, households, building_types). """ parcel_ids = job_dataset.get_attribute("parcel_id") building_ids = job_dataset.get_attribute("building_id") building_types = job_dataset.get_attribute("building_type") try: impute_sqft_flags = job_dataset.get_attribute("impute_building_sqft_flag") except: impute_sqft_flags = zeros(job_dataset.size()) is_considered = logical_and(parcel_ids > 0, building_ids <= 0) # jobs that have assigned parcel but not building job_index_home_based = where(logical_and(is_considered, building_types == 1))[0] job_index_governmental = where(logical_and(is_considered, building_types == 3))[0] building_dataset = dataset_pool.get_dataset('building') parcel_ids_in_bldgs = building_dataset.get_attribute("parcel_id") bldg_ids_in_bldgs = building_dataset.get_id_attribute() bldg_types_in_bldgs = building_dataset.get_attribute("building_type_id") non_res_sqft = building_dataset.get_attribute("non_residential_sqft") occupied = building_dataset.compute_variables(["urbansim_parcel.building.occupied_building_sqft_by_jobs"], dataset_pool=dataset_pool) is_governmental = building_dataset.compute_variables(["building.disaggregate(building_type.generic_building_type_id == 7)"], dataset_pool=dataset_pool) # assign buildings to governmental jobs randomly unique_parcels = unique(parcel_ids[job_index_governmental]) logger.log_status("Placing governmental jobs ...") for parcel in unique_parcels: idx_in_bldgs = where(parcel_ids_in_bldgs[is_governmental] == parcel)[0] if idx_in_bldgs.size <= 0: continue idx_in_jobs = where(parcel_ids[job_index_governmental] == parcel)[0] draw = sample_replace(idx_in_bldgs, idx_in_jobs.size) building_ids[job_index_governmental[idx_in_jobs]] = bldg_ids_in_bldgs[where(is_governmental)[0][draw]] logger.log_status("%s governmental jobs (out of %s gov. jobs) were placed." % ( (building_ids[job_index_governmental]>0).sum(), job_index_governmental.size)) logger.log_status("The not-placed governmental jobs will be added to the non-home based jobs.") # consider the unplaced governmental jobs together with other non-home-based jobs is_now_considered = logical_and(is_considered, building_ids <= 0) job_index_non_home_based = where(logical_and(is_now_considered, logical_or(building_types == 2, building_types == 3)))[0] # assign buildings to non_home_based jobs based on available space unique_parcels = unique(parcel_ids[job_index_non_home_based]) job_building_types = job_dataset.compute_variables(["bldgs_building_type_id = job.disaggregate(building.building_type_id)"], dataset_pool=dataset_pool) where_valid_jbt = where(logical_and(job_building_types>0, logical_or(building_types == 2, building_types==3)))[0] building_type_dataset = dataset_pool.get_dataset("building_type") available_building_types= building_type_dataset.get_id_attribute() idx_available_bt = building_type_dataset.get_id_index(available_building_types) sectors = job_dataset.get_attribute("sector_id") unique_sectors = unique(sectors) sector_bt_distribution = zeros((unique_sectors.size, building_type_dataset.size()), dtype="float32") jobs_sqft = job_dataset.get_attribute_by_index("sqft", job_index_non_home_based).astype("float32") job_dataset._compute_if_needed("urbansim_parcel.job.zone_id", dataset_pool=dataset_pool) jobs_zones = job_dataset.get_attribute_by_index("zone_id", job_index_non_home_based) new_jobs_sqft = job_dataset.get_attribute("sqft").copy() # find sector -> building_type distribution sector_index_mapping = {} for isector in range(unique_sectors.size): idx = where(sectors[where_valid_jbt]==unique_sectors[isector])[0] if idx.size == 0: continue o = ones(idx.size, dtype="int32") sector_bt_distribution[isector,:] = ndimage_sum(o, labels=job_building_types[where_valid_jbt[idx]], index=available_building_types) sector_bt_distribution[isector,:] = sector_bt_distribution[isector,:]/sector_bt_distribution[isector,:].sum() sector_index_mapping[unique_sectors[isector]] = isector # create a lookup table for zonal average per building type of sqft per employee zone_average_dataset = dataset_pool.get_dataset("building_sqft_per_job") zone_bt_lookup = zone_average_dataset.get_building_sqft_as_table(job_dataset.get_attribute("zone_id").max(), available_building_types.max()) counter_zero_capacity = 0 counter_zero_distr = 0 # iterate over parcels logger.log_status("Placing non-home-based jobs ...") for parcel in unique_parcels: idx_in_bldgs = where(parcel_ids_in_bldgs == parcel)[0] if idx_in_bldgs.size <= 0: continue idx_in_jobs = where(parcel_ids[job_index_non_home_based] == parcel)[0] capacity = maximum(non_res_sqft[idx_in_bldgs] - occupied[idx_in_bldgs],0) #capacity = non_res_sqft[idx_in_bldgs] - occupied[idx_in_bldgs] if capacity.sum() <= 0: counter_zero_capacity += idx_in_jobs.size continue this_jobs_sectors = sectors[job_index_non_home_based][idx_in_jobs] this_jobs_sqft_table = resize(jobs_sqft[idx_in_jobs], (idx_in_bldgs.size, idx_in_jobs.size)) wn = jobs_sqft[idx_in_jobs] <= 0 for i in range(idx_in_bldgs.size): this_jobs_sqft_table[i, where(wn)[0]] = zone_bt_lookup[jobs_zones[idx_in_jobs[wn]], bldg_types_in_bldgs[idx_in_bldgs[i]]] #supply_demand_ratio = (resize(capacity, (capacity.size, 1))/this_jobs_sqft_table.astype("float32").sum(axis=0))/float(idx_in_jobs.size)*0.9 supply_demand_ratio = resize(capacity, (capacity.size, 1))/(this_jobs_sqft_table.astype("float32")*float(idx_in_jobs.size))*0.9 if any(supply_demand_ratio < 1): # applies only if supply is smaller than demand for i in range(idx_in_bldgs.size): if any(supply_demand_ratio[i,:] < 1): this_jobs_sqft_table[i,:] = this_jobs_sqft_table[i,:] * supply_demand_ratio[i,:] probcomb = zeros(this_jobs_sqft_table.shape) bt = bldg_types_in_bldgs[idx_in_bldgs] ibt = building_type_dataset.get_id_index(bt) for i in range(probcomb.shape[0]): for j in range(probcomb.shape[1]): probcomb[i,j] = sector_bt_distribution[sector_index_mapping[this_jobs_sectors[j]],ibt[i]] pcs = probcomb.sum(axis=0) probcomb = probcomb/pcs wz = where(pcs<=0)[0] counter_zero_distr += wz.size probcomb[:, wz] = 0 # to avoid nan values taken = zeros(capacity.shape) has_sqft = this_jobs_sqft_table > 0 while True: if (has_sqft * probcomb).sum() <= 0: break req = (this_jobs_sqft_table * probcomb).sum(axis=0) maxi = req.max() wmaxi = where(req==maxi)[0] drawjob = sample_noreplace(arange(wmaxi.size), 1) # draw job from jobs with the maximum size imax_req = wmaxi[drawjob] weights = has_sqft[:,imax_req] * probcomb[:,imax_req] draw = probsample_noreplace(arange(probcomb.shape[0]), 1, resize(weights/weights.sum(), (probcomb.shape[0],))) if (taken[draw] + this_jobs_sqft_table[draw,imax_req]) > capacity[draw]: probcomb[draw,imax_req]=0 continue taken[draw] = taken[draw] + this_jobs_sqft_table[draw,imax_req] building_ids[job_index_non_home_based[idx_in_jobs[imax_req]]] = bldg_ids_in_bldgs[idx_in_bldgs[draw]] probcomb[:,imax_req] = 0 new_jobs_sqft[job_index_non_home_based[idx_in_jobs[imax_req]]] = int(min(self.maximum_sqft, max(round(this_jobs_sqft_table[draw,imax_req]), self.minimum_sqft))) logger.log_status("%s non home based jobs (out of %s nhb jobs) were placed." % ( (building_ids[job_index_non_home_based]>0).sum(), job_index_non_home_based.size)) logger.log_status("Unplaced due to zero capacity: %s" % counter_zero_capacity) logger.log_status("Unplaced due to zero distribution: %s" % counter_zero_distr) job_dataset.modify_attribute(name="building_id", data = building_ids) # re-classify unplaced non-home based jobs to home-based if parcels contain residential buildings bldgs_is_residential = logical_and(logical_not(is_governmental), building_dataset.compute_variables(["urbansim_parcel.building.is_residential"], dataset_pool=dataset_pool)) is_now_considered = logical_and(parcel_ids > 0, building_ids <= 0) job_index_non_home_based_unplaced = where(logical_and(is_now_considered, building_types == 2))[0] unique_parcels = unique(parcel_ids[job_index_non_home_based_unplaced]) imputed_sqft = 0 logger.log_status("Try to reclassify non-home-based jobs (excluding governmental jobs) ...") for parcel in unique_parcels: idx_in_bldgs = where(parcel_ids_in_bldgs == parcel)[0] if idx_in_bldgs.size <= 0: continue idx_in_jobs = where(parcel_ids[job_index_non_home_based_unplaced] == parcel)[0] where_residential = where(bldgs_is_residential[idx_in_bldgs])[0] if where_residential.size > 0: building_types[job_index_non_home_based_unplaced[idx_in_jobs]] = 1 # set to home-based jobs elif non_res_sqft[idx_in_bldgs].sum() <= 0: # impute non_residential_sqft and assign buildings this_jobs_sectors = sectors[job_index_non_home_based_unplaced][idx_in_jobs] this_jobs_sqft_table = resize(jobs_sqft[idx_in_jobs], (idx_in_bldgs.size, idx_in_jobs.size)) wn = jobs_sqft[idx_in_jobs] <= 0 for i in range(idx_in_bldgs.size): this_jobs_sqft_table[i, where(wn)[0]] = zone_bt_lookup[jobs_zones[idx_in_jobs[wn]], bldg_types_in_bldgs[idx_in_bldgs[i]]] probcomb = zeros(this_jobs_sqft_table.shape) bt = bldg_types_in_bldgs[idx_in_bldgs] ibt = building_type_dataset.get_id_index(bt) for i in range(probcomb.shape[0]): for j in range(probcomb.shape[1]): probcomb[i,j] = sector_bt_distribution[sector_index_mapping[this_jobs_sectors[j]],ibt[i]] for ijob in range(probcomb.shape[1]): if (probcomb[:,ijob].sum() <= 0) or (impute_sqft_flags[job_index_non_home_based_unplaced[ijob]] == 0): continue weights = probcomb[:,ijob] draw = probsample_noreplace(arange(probcomb.shape[0]), 1, resize(weights/weights.sum(), (probcomb.shape[0],))) non_res_sqft[idx_in_bldgs[draw]] += this_jobs_sqft_table[draw,ijob] imputed_sqft += this_jobs_sqft_table[draw,ijob] building_ids[job_index_non_home_based_unplaced[idx_in_jobs[ijob]]] = bldg_ids_in_bldgs[idx_in_bldgs[draw]] new_jobs_sqft[job_index_non_home_based[idx_in_jobs[ijob]]] = int(min(self.maximum_sqft, max(round(this_jobs_sqft_table[draw,ijob]), self.minimum_sqft))) building_dataset.modify_attribute(name="non_residential_sqft", data = non_res_sqft) job_dataset.modify_attribute(name="building_id", data = building_ids) job_dataset.modify_attribute(name="building_type", data = building_types) job_dataset.modify_attribute(name="sqft", data = new_jobs_sqft) old_nhb_size = job_index_non_home_based.size job_index_home_based = where(logical_and(is_considered, building_types == 1))[0] job_index_non_home_based = where(logical_and(is_considered, building_types == 2))[0] logger.log_status("%s non-home based jobs reclassified as home-based." % (old_nhb_size-job_index_non_home_based.size)) logger.log_status("%s non-residential sqft imputed." % imputed_sqft) logger.log_status("Additionaly, %s non home based jobs were placed due to imputed sqft." % \ (building_ids[job_index_non_home_based_unplaced]>0).sum()) # home_based jobs unique_parcels = unique(parcel_ids[job_index_home_based]) capacity_in_buildings = building_dataset.compute_variables([ "urbansim_parcel.building.vacant_home_based_job_space"], dataset_pool=dataset_pool) parcels_with_exceeded_capacity = [] # iterate over parcels logger.log_status("Placing home-based jobs ...") for parcel in unique_parcels: idx_in_bldgs = where(parcel_ids_in_bldgs == parcel)[0] idx_in_jobs = where(parcel_ids[job_index_home_based] == parcel)[0] capacity = capacity_in_buildings[idx_in_bldgs] if capacity.sum() <= 0: continue probcomb = ones((idx_in_bldgs.size, idx_in_jobs.size)) taken = zeros(capacity.shape, dtype="int32") while True: zero_cap = where((capacity - taken) <= 0)[0] probcomb[zero_cap,:] = 0 if probcomb.sum() <= 0: break req = probcomb.sum(axis=0) wmaxi = where(req==req.max())[0] drawjob = sample_noreplace(arange(wmaxi.size), 1) # draw job from available jobs imax_req = wmaxi[drawjob] weights = probcomb[:,imax_req] # sample building draw = probsample_noreplace(arange(probcomb.shape[0]), 1, resize(weights/weights.sum(), (probcomb.shape[0],))) taken[draw] = taken[draw] + 1 building_ids[job_index_home_based[idx_in_jobs[imax_req]]] = bldg_ids_in_bldgs[idx_in_bldgs[draw]] probcomb[:,imax_req] = 0 if -1 in building_ids[job_index_home_based[idx_in_jobs]]: parcels_with_exceeded_capacity.append(parcel) parcels_with_exceeded_capacity = array(parcels_with_exceeded_capacity) logger.log_status("%s home based jobs (out of %s hb jobs) were placed." % ((building_ids[job_index_home_based]>0).sum(), job_index_home_based.size)) # assign building type where missing # determine regional distribution idx_home_based = where(building_types == 1)[0] idx_non_home_based = where(building_types == 2)[0] idx_bt_missing = where(building_types <= 0)[0] if idx_bt_missing.size > 0: # sample building types sample_bt = probsample_replace(array([1,2]), idx_bt_missing.size, array([idx_home_based.size, idx_non_home_based.size])/float(idx_home_based.size + idx_non_home_based.size)) # coerce to int32 (on a 64 bit machine, sample_bt will be of type int64) building_types[idx_bt_missing] = sample_bt.astype(int32) job_dataset.modify_attribute(name="building_type", data = building_types) if out_storage is not None: job_dataset.write_dataset(out_table_name=jobs_table, out_storage=out_storage, attributes=AttributeType.PRIMARY) building_dataset.write_dataset(out_table_name='buildings', out_storage=out_storage, attributes=AttributeType.PRIMARY) logger.log_status("Assigning building_id to jobs done.")