def compute(self, dataset_pool): vl = self.get_dataset().get_attribute("vacant_land_sqft").astype( "float32") return safe_array_divide( self.get_dataset().sum_dataset_over_ids( dataset_pool.get_dataset('gridcell'), "total_value_vacant_land"), vl)
def compute(self, dataset_pool): buildings = self.get_dataset() non_residential_sqft = buildings.get_attribute("non_residential_sqft") building_sqft_per_job = buildings.get_attribute("building_sqft_per_job") job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job']) job_spaces[buildings['year_built']<=2000] = buildings['job_capacity'][buildings['year_built']<=2000] return job_spaces
def compute(self, dataset_pool): zone_set = self.get_dataset() travel_data = dataset_pool.get_dataset('travel_data') from_zone_id = travel_data.get_attribute('from_zone_id') zone_ids = zone_set.get_attribute('zone_id') time = travel_data.get_attribute(self.time_attribute_name) trips = travel_data.get_attribute(self.trips_attribute_name) numerator = array(ndimage_sum(time * trips, labels = from_zone_id, index=zone_ids)) denominator = array(ndimage_sum(trips, labels = from_zone_id, index=zone_ids), dtype=float32) # if there is a divide by zero then subsititute the values from the zone one below that one # if there are contigious places of zero division the values should propigate upon iteration no_trips_from_here = where(denominator == 0)[0] while no_trips_from_here.size != 0: if no_trips_from_here.size == denominator.size: logger.log_warning("%s attribute of travel_data is all zeros; %s returns all zeros" % (self.trips_attribute_name, self.name() )) break substitute_locations = no_trips_from_here - 1 # a mapping, what zone the new data will come from if substitute_locations[0] < 0: substitute_locations[0] = 1 numerator[no_trips_from_here] = numerator[substitute_locations] denominator[no_trips_from_here] = denominator[substitute_locations] no_trips_from_here = where(denominator == 0)[0] return safe_array_divide(numerator, denominator)
def compute(self, dataset_pool): """ use networkx to determine whether two geographies are adjacent with k degree of expansion """ zones = self.get_dataset() adjacent_zones = dataset_pool.get_dataset('adjacent_zone') id_max = zones['zone_id'].max()+1 W = zeros((id_max, id_max), dtype='int8') #diagonal sum_income = zeros(id_max, dtype=zones['sum_income'].dtype) sum_households = zeros(id_max, dtype=zones['sum_households'].dtype) sum_income[zones['zone_id']] = zones['sum_income'] sum_households[zones['zone_id']] = zones['sum_households'] if not nx: raise ImportError, "networkx module is required." G = nx.Graph() G.add_nodes_from(zones['zone_id']) G.add_edges_from(adjacent_zones.get_multiple_attributes(['zone_id', 'adjacent_zone_id'])) length = nx.all_pairs_shortest_path_length(G, cutoff=self.order) for key, val in length.items(): W[key][val.keys()] = 1 sum_income = dot(W, sum_income[:, newaxis])[:, 0] sum_households = dot(W, sum_households[:, newaxis])[:, 0] results = safe_array_divide(sum_income, sum_households.astype('f')) return results[zones['zone_id']]
def compute_building_type_distribution(self, building_type_dataset, realestate_dataset_name, regions): parcels = self.dataset_pool.get_dataset('parcel') building_type_ids = building_type_dataset.get_id_attribute() regions.compute_variables(map(lambda type: "units_proposed_for_bt_%s = %s.aggregate(psrc_parcel.parcel.units_proposed_for_building_type_%s)" % (type, self.subarea_name, type), building_type_ids[logical_not(in1d(building_type_ids, self.bt_do_not_count))]), dataset_pool=self.dataset_pool) sumunits_residential = zeros(regions.size(), dtype='float32') sumunits_nonresidential = zeros(regions.size(), dtype='float32') for itype in range(building_type_ids.size): if building_type_ids[itype] in self.bt_do_not_count: continue potential_units = regions["units_proposed_for_bt_%s" % building_type_ids[itype]] if building_type_dataset['is_residential'][itype]: sumunits_residential = sumunits_residential + potential_units else: sumunits_nonresidential = sumunits_nonresidential + potential_units building_type_distribution = {} for itype in range(building_type_ids.size): building_type_distribution[itype] = zeros(regions.size()) if building_type_ids[itype] in self.bt_do_not_count: continue if building_type_dataset['is_residential'][itype]: sumunits = sumunits_residential else: sumunits = sumunits_nonresidential building_type_distribution[itype] = safe_array_divide(regions['units_proposed_for_bt_%s' % building_type_ids[itype]], sumunits) return building_type_distribution
def compute(self, dataset_pool): zones = self.get_dataset() zone_ids = zones.get_id_attribute() travel_data = dataset_pool.get_dataset("travel_data") # import ipdb; ipdb.set_trace() if self.aggregate_by_origin: attribute_zone_id = travel_data.get_attribute("from_zone_id") summary_zone_id = travel_data.get_attribute("to_zone_id") else: attribute_zone_id = travel_data.get_attribute("to_zone_id") summary_zone_id = travel_data.get_attribute("from_zone_id") zone_index = zones.get_id_index(attribute_zone_id) attribute = zones[self.zone_attribute_to_access][zone_index] weight = safe_array_divide(1.0, power(travel_data[self.travel_data_attribute],2)) function = getattr(ndimage, self.function) results = array(function(attribute * weight, labels = summary_zone_id, index=zone_ids)) return results ## unittests in child class
def _delete(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool): """similar to subtract action, instead of unplacing agents delete remove agents from the agent dataset, those agents won't be available for later action """ fit_index = self.get_fit_agents_index( agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if amount > fit_index.size or amount < 0: logger.log_warning("Refinement requests to delete %i agents, but there are %i agents in total satisfying %s;" \ "delete %i agents instead" % (amount, fit_index.size, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip(' and '), fit_index.size) ) amount = fit_index.size if amount == fit_index.size: movers_index = fit_index else: movers_index = sample_noreplace(fit_index, amount) agents_pool = list(set(agents_pool) - set(movers_index)) ## modify location capacity attribute if specified if this_refinement.location_capacity_attribute is not None and len( this_refinement.location_capacity_attribute) > 0: location_dataset = dataset_pool.get_dataset( VariableName( this_refinement.location_expression).get_dataset_name()) movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0])[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id) # see previous comment about histogram function num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() + 1))[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ (location_dataset.dataset_name, agent_dataset.dataset_name), dataset_pool=dataset_pool) shrink_factor = safe_array_divide( (num_of_agents_by_location - num_of_movers_by_location).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero=1.0) new_values = round_(shrink_factor * location_dataset.get_attribute( this_refinement.location_capacity_attribute)) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values) self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index) agent_dataset.remove_elements(array(movers_index))
def compute(self, dataset_pool): buildings = self.get_dataset() non_residential_sqft = buildings.get_attribute("non_residential_sqft") building_sqft_per_job = buildings.get_attribute("building_sqft_per_job") #job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job']) job_spaces = safe_array_divide(buildings['non_residential_sqft'], 250) ## only do this when job_capacity and year_built are primary attributes of buildings return job_spaces
def _subtract(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool ): fit_index = self.get_fit_agents_index(agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if amount > fit_index.size: logger.log_warning("Refinement requests to subtract %i agents, but there are %i agents in total satisfying %s;" \ "subtract %i agents instead" % (amount, fit_index.size, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip(' and '), fit_index.size) ) amount = fit_index.size if amount == fit_index.size: movers_index = fit_index else: movers_index = sample_noreplace( fit_index, amount ) agents_pool += movers_index.tolist() ## modify location capacity attribute if specified if this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0: location_dataset = dataset_pool.get_dataset( VariableName( this_refinement.location_expression ).get_dataset_name() ) movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id ) # backward compatability code for older versions of numpy -- no longer required since we need numpy 1.2.1 or greater # new=False argument to histogram tells it to use deprecated behavior for now (to be removed in numpy 1.3) # See numpy release notes -- search for histogram # if numpy.__version__ >= '1.2.0': # num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()), new=False)[0] # else: # num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()))[0] num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ (location_dataset.dataset_name, agent_dataset.dataset_name), dataset_pool=dataset_pool) shrink_factor = safe_array_divide( (num_of_agents_by_location - num_of_movers_by_location ).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 ) new_values = round_( shrink_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) ) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values ) self._add_refinement_info_to_dataset(location_dataset, ("refinement_id", "transaction_id"), this_refinement, index=movers_location_index) agent_dataset.modify_attribute(location_dataset.get_id_name()[0], -1 * ones( movers_index.size, dtype='int32' ), index = movers_index ) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=movers_index)
def compute(self, dataset_pool): submarket = self.get_dataset() uptake = clip(submarket['households'] - submarket['households_lag1'], 0, inf) vacant_units = clip(submarket['residential_units'] - submarket['households'], 0, inf) results = safe_array_divide( uptake, vacant_units.astype('f') ) results = clip(results, self.lower_bound, self.upper_bound) all_units_occupied = logical_and(vacant_units==0, submarket['households'] > 0) results[all_units_occupied] = self.upper_bound return results
def compute(self, dataset_pool): buildings = self.get_dataset() non_residential_sqft = buildings.get_attribute("non_residential_sqft") building_sqft_per_job = buildings.get_attribute( "building_sqft_per_job") job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job']) job_spaces[buildings['year_built'] <= 2000] = buildings[ 'job_capacity'][buildings['year_built'] <= 2000] return job_spaces
def compute_sdratio(self, submarkets, choice_set, capacity_string, demand_string): #choice_submarket_id = choice_set.compute_variables("urbansim_parcel.%s.%s" % (choice_set.get_dataset_name(), submarkets.get_id_name()[0])) choice_submarket_id = choice_set.get_attribute("building_id") choice_submarket_index = submarkets.get_id_index( choice_submarket_id ) supply = submarkets.sum_over_ids( choice_submarket_id, self.capacity ) demand = submarkets.compute_variables("demand=submarket.aggregate(%s.%s)" % (choice_set.get_dataset_name(), demand_string) ) ##TODO: an adjustment ratio should be included since this variable only represents the demand from agents in this chunk sdratio_submarket = safe_array_divide(supply, demand) return (sdratio_submarket, sdratio_submarket[choice_submarket_index] )
def compute(self, dataset_pool): nb = self.get_dataset().get_attribute(self.number_of_buildings) buildings = dataset_pool.get_dataset('building') age = buildings.get_attribute(self.building_age) mask = ma.getmask(age) if mask is ma.nomask: donotcount=0 else: # count number of buildings that have missing year_built and therefore won't be included in the average. donotcount = self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"), mask.astype("int8")) # fill -1 for masked values, in order to distinguish it from age=0. return safe_array_divide(self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"), age), nb-donotcount, -1)
def _delete(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool ): """similar to subtract action, instead of unplacing agents delete remove agents from the agent dataset, those agents won't be available for later action """ fit_index = self.get_fit_agents_index(agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if amount > fit_index.size or amount < 0: logger.log_warning("Refinement requests to delete %i agents, but there are %i agents in total satisfying %s;" \ "delete %i agents instead" % (amount, fit_index.size, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip(' and '), fit_index.size) ) amount = fit_index.size if amount == fit_index.size: movers_index = fit_index else: movers_index = sample_noreplace( fit_index, amount ) agents_pool = list( set(agents_pool) - set(movers_index) ) ## modify location capacity attribute if specified if this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0: location_dataset = dataset_pool.get_dataset( VariableName( this_refinement.location_expression ).get_dataset_name() ) movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id ) # see previous comment about histogram function num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ (location_dataset.dataset_name, agent_dataset.dataset_name), dataset_pool=dataset_pool) shrink_factor = safe_array_divide( (num_of_agents_by_location - num_of_movers_by_location ).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 ) new_values = round_( shrink_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) ) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values ) self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index) agent_dataset.remove_elements( array(movers_index) )
def compute_job_building_type_distribution(self, building_type_dataset, realestate_dataset_name, regions): building_type_ids = building_type_dataset.get_id_attribute() regions.compute_variables(map(lambda type: "number_of_jobs_for_bt_%s = %s.aggregate(psrc_parcel.%s.number_of_non_home_based_jobs * (%s.building_type_id == %s))" % (type, self.subarea_name, realestate_dataset_name, realestate_dataset_name, type), building_type_ids[logical_not(in1d(building_type_ids, self.bt_do_not_count))]) + ["number_of_nhb_jobs = %s.aggregate(psrc_parcel.%s.number_of_non_home_based_jobs)" % (self.subarea_name, realestate_dataset_name)], dataset_pool=self.dataset_pool) building_type_distribution = {} for itype in range(building_type_ids.size): building_type_distribution[itype] = zeros(regions.size()) if building_type_ids[itype] in self.bt_do_not_count: continue building_type_distribution[itype] = safe_array_divide( regions['number_of_jobs_for_bt_%s' % building_type_ids[itype]], regions['number_of_nhb_jobs']) return building_type_distribution
def compute(self, dataset_pool): buildings = self.get_dataset() non_residential_sqft = buildings.get_attribute("non_residential_sqft") building_sqft_per_job = buildings.get_attribute("building_sqft_per_job") job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job']) ## only do this when job_capacity and year_built are primary attributes of buildings known_names = buildings.get_known_attribute_names() if 'job_capacity' in known_names: # and 'year_built' in known_names: self.add_and_solve_dependencies(["building.job_capacity"#, "building.year_built" ], dataset_pool=dataset_pool) #base_year = SimulationState().get_start_time() #base_year = 2000 # This can't be taken from SimulationState because of running refinement with different base year #job_spaces[buildings['year_built']<=base_year] = buildings['job_capacity'][buildings['year_built']<=base_year] # Rather than using year_built we can do the following because the building construction model assigns -1 to new buildings job_spaces[buildings['job_capacity'] >= 0] = buildings['job_capacity'][buildings['job_capacity'] >= 0] return job_spaces
def compute(self, dataset_pool): nb = self.get_dataset().get_attribute(self.number_of_buildings) buildings = dataset_pool.get_dataset('building') age = buildings.get_attribute( self.building_age) * buildings.get_attribute(self.is_building_type) mask = ma.getmask(age) if mask is ma.nomask: donotcount = 0 else: mask = mask * buildings.get_attribute(self.is_building_type) # count number of buildings that have missing year_built and therefore won't be included in the average. donotcount = self.get_dataset().sum_over_ids( buildings.get_attribute("grid_id"), mask.astype("int8")) # fill -1 for masked values, in order to distinguish it from age=0. return safe_array_divide( self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"), age), nb - donotcount, -1)
def compute(self, dataset_pool): proposals = self.get_dataset() unit_price = proposals.get_attribute("unit_price") existing_units = proposals.get_attribute("existing_units") improvement_value = proposals.get_attribute("improvement_value") land_area_taken = proposals.get_attribute("land_area_taken") land_area = proposals.get_attribute("land_area") ## is_redevelopment is optional, if is_redevelopment is not one of proposal attributes, default to 0 if 'is_redevelopment' in proposals.get_known_attribute_names(): is_redevelopment = proposals.get_attribute('is_redevelopment') else: is_redevelopment = 0 results = safe_array_divide( (unit_price * existing_units - improvement_value), land_area) * land_area_taken + improvement_value * is_redevelopment return results
def compute(self, dataset_pool): ds = self.get_dataset() return safe_array_divide(ds.get_attribute(self.vacant_units), ds.get_attribute(self.number_of_units))
def compute(self, dataset_pool): ds = self.get_dataset() alldata = dataset_pool.get_dataset('alldata') results = ds['M_{0}j0'.format(self.type_id)] * safe_array_divide(alldata['TM_{0}t'.format(self.type_id)], alldata['TM_{0}0'.format(self.type_id)]) * safe_array_divide(ds['V_ijt'], ds['V_ij0']) return results
def compute(self, dataset_pool): nh = self.get_dataset().get_attribute(self.number_of_households) return safe_array_divide( self.get_dataset().get_attribute(self.population), nh)
def compute(self, dataset_pool): ds = self.get_dataset() values_sqft_per_job = ds.get_attribute(self.sqft_per_job) values_sqft = ds.get_attribute(self.sqft) return safe_array_divide(values_sqft, values_sqft_per_job, type="int32")
def compute(self, dataset_pool): sp = self.get_dataset().get_attribute("%s_space" % self.type) return safe_array_divide(self.get_dataset().get_attribute("total_value_%s" % self.type), sp)
def compute(self, dataset_pool): acres = self.get_dataset().get_attribute(self.acres) return safe_array_divide( self.get_dataset().get_attribute(self.land_value), acres)
def compute(self, dataset_pool): nou = self.get_dataset().get_attribute("buildings_%s_space_where_total_value" % self.type) return safe_array_divide(self.get_dataset().get_attribute( "total_value_%s" % self.type), nou)
def compute(self, dataset_pool): return 100 * safe_array_divide( self.get_dataset().get_attribute(self.low_income), self.get_dataset().get_attribute(self.number_of_households))
def _add(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool ): fit_index = self.get_fit_agents_index(agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) movers_index = array([],dtype="int32") amount_from_agents_pool = min( amount, len(agents_pool) ) if amount_from_agents_pool > 0: agents_index_from_agents_pool = sample_noreplace( agents_pool, amount_from_agents_pool ) [ agents_pool.remove(i) for i in agents_index_from_agents_pool ] if fit_index.size == 0: ##cannot find agents to copy their location or clone them, place agents in agents_pool if amount > amount_from_agents_pool: logger.log_warning("Refinement requests to add %i agents, but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \ "add %i agents instead" % (amount, amount_from_agents_pool, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression]).strip(' and '), amount_from_agents_pool,) ) amount = amount_from_agents_pool # sample from all suitable locations is_suitable_location = location_dataset.compute_variables( this_refinement.location_expression, dataset_pool=dataset_pool ) location_id_for_agents_pool = sample_replace( location_dataset.get_id_attribute()[is_suitable_location], amount_from_agents_pool ) else: #sample from locations of suitable agents agents_index_for_location = sample_replace( fit_index, amount_from_agents_pool) location_id_for_agents_pool = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[agents_index_for_location] movers_index = concatenate( (movers_index, agents_index_for_location) ) elif fit_index.size == 0: ## no agents in agents_pool and no agents to clone either, --> fail logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \ ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') ) return if amount > amount_from_agents_pool: agents_index_to_clone = sample_replace( fit_index, amount - amount_from_agents_pool) movers_index = concatenate( (movers_index, agents_index_to_clone) ) if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0: movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id ) # see previous comment about histogram function num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ ( location_dataset.dataset_name, agent_dataset.dataset_name ), dataset_pool=dataset_pool) expand_factor = safe_array_divide( (num_of_agents_by_location + num_of_movers_by_location ).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 ) new_values = round_( expand_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) ) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values ) self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index) if amount_from_agents_pool > 0: agent_dataset.modify_attribute( location_dataset.get_id_name()[0], location_id_for_agents_pool, agents_index_from_agents_pool ) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_from_agents_pool) if amount > amount_from_agents_pool: new_agents_index = agent_dataset.duplicate_rows(agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=new_agents_index)
def _subtract(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool): fit_index = self.get_fit_agents_index( agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if amount > fit_index.size: logger.log_warning("Refinement requests to subtract %i agents, but there are %i agents in total satisfying %s;" \ "subtract %i agents instead" % (amount, fit_index.size, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip(' and '), fit_index.size) ) amount = fit_index.size if amount == fit_index.size: movers_index = fit_index else: movers_index = sample_noreplace(fit_index, amount) agents_pool += movers_index.tolist() ## modify location capacity attribute if specified if this_refinement.location_capacity_attribute is not None and len( this_refinement.location_capacity_attribute) > 0: location_dataset = dataset_pool.get_dataset( VariableName( this_refinement.location_expression).get_dataset_name()) movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0])[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id) # backward compatability code for older versions of numpy -- no longer required since we need numpy 1.2.1 or greater # new=False argument to histogram tells it to use deprecated behavior for now (to be removed in numpy 1.3) # See numpy release notes -- search for histogram # if numpy.__version__ >= '1.2.0': # num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()), new=False)[0] # else: # num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()))[0] num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() + 1))[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ (location_dataset.dataset_name, agent_dataset.dataset_name), dataset_pool=dataset_pool) shrink_factor = safe_array_divide( (num_of_agents_by_location - num_of_movers_by_location).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero=1.0) new_values = round_(shrink_factor * location_dataset.get_attribute( this_refinement.location_capacity_attribute)) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values) self._add_refinement_info_to_dataset( location_dataset, ("refinement_id", "transaction_id"), this_refinement, index=movers_location_index) agent_dataset.modify_attribute(location_dataset.get_id_name()[0], -1 * ones(movers_index.size, dtype='int32'), index=movers_index) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=movers_index)
def compute(self, dataset_pool): return safe_array_divide(self.get_dataset().get_attribute(self.building_units), self.get_dataset().get_attribute(self.number_of_units))
def compute(self, dataset_pool): ds = self.get_dataset() return safe_array_divide(ds.get_attribute(self.population), ds.get_attribute(self.acres_of_land))
def compute(self, dataset_pool): nh = self.get_dataset().get_attribute(self.number_of_households) return safe_array_divide(self.get_dataset().get_attribute(self.population), nh)
def _add(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool ): fit_index = self.get_fit_agents_index(agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if this_refinement.agent_expression is not None and len(this_refinement.agent_expression) > 0: agents_index = where(agent_dataset.compute_variables(this_refinement.agent_expression, dataset_pool=dataset_pool)>0)[0] else: agents_index = arange(agent_dataset.size()) movers_index = array([],dtype="int32") ar_pool = array(agents_pool) fitted_agents_pool = ar_pool[in1d(ar_pool, agents_index)] amount_from_agents_pool = min( amount, fitted_agents_pool.size ) prob_string = self.probability_attributes.get(agent_dataset.get_dataset_name(),None) if prob_string is not None: probs_values = (agent_dataset.compute_variables([prob_string], dataset_pool=dataset_pool)).astype('int32') uprobs_values = unique(probs_values[fit_index]) if uprobs_values.size > 0: probs_existing = array(ndimage_sum(ones(fit_index.size), labels=probs_values[fit_index], index=uprobs_values)) if amount_from_agents_pool > 0: if prob_string is not None and uprobs_values.size > 0: prob_pool_values = probs_values[fitted_agents_pool] probs_pool=zeros(prob_pool_values.size) for i in range(uprobs_values.size): probpoolidx = where(prob_pool_values == uprobs_values[i])[0] if probpoolidx.size == 0: continue probs_pool[probpoolidx]=probs_existing[i]/float(probpoolidx.size) probs_pool[probs_pool<=0] = (probs_existing.min()/10.0)/float((probs_pool<=0).sum()) else: probs_pool=ones(fitted_agents_pool.size) agents_index_from_agents_pool = probsample_noreplace( fitted_agents_pool, amount_from_agents_pool, prob_array=probs_pool ) [ agents_pool.remove(i) for i in agents_index_from_agents_pool ] if fit_index.size == 0: ##cannot find agents to copy their location or clone them, place agents in agents_pool if amount > amount_from_agents_pool: logger.log_warning("Refinement requests to add %i agents, but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \ "add %i agents instead" % (amount, amount_from_agents_pool, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression]).strip(' and '), amount_from_agents_pool,) ) amount = amount_from_agents_pool # sample from all suitable locations is_suitable_location = location_dataset.compute_variables( this_refinement.location_expression, dataset_pool=dataset_pool ) location_id_for_agents_pool = sample_replace( location_dataset.get_id_attribute()[is_suitable_location], amount_from_agents_pool ) else: #sample from locations of suitable agents agents_index_for_location = sample_replace( fit_index, amount_from_agents_pool) location_id_for_agents_pool = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[agents_index_for_location] movers_index = concatenate( (movers_index, agents_index_for_location) ) elif fit_index.size == 0: ## no agents in agents_pool and no agents to clone either, --> fail logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \ ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') ) return if amount > amount_from_agents_pool: agents_index_to_clone = sample_replace( fit_index, amount - amount_from_agents_pool) movers_index = concatenate( (movers_index, agents_index_to_clone) ) if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0: movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id ) # see previous comment about histogram function num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ ( location_dataset.dataset_name, agent_dataset.dataset_name ), dataset_pool=dataset_pool) expand_factor = safe_array_divide( (num_of_agents_by_location + num_of_movers_by_location ).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 ) new_values = round_( expand_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) ) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values ) self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index) if amount_from_agents_pool > 0: agent_dataset.modify_attribute( 'building_id', -1 * ones( agents_index_from_agents_pool.size, dtype='int32' ), agents_index_from_agents_pool ) agent_dataset.modify_attribute( location_dataset.get_id_name()[0], location_id_for_agents_pool, agents_index_from_agents_pool ) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_from_agents_pool) self.processed_locations['add'] = concatenate((self.processed_locations.get('add', array([])), unique(location_dataset[self.subarea_id_name][location_dataset.get_id_index(location_id_for_agents_pool)]))) if amount > amount_from_agents_pool: new_agents_index = agent_dataset.duplicate_rows(agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=new_agents_index) if location_dataset.get_dataset_name() <> 'building': agent_dataset.modify_attribute( 'building_id', -1 * ones( new_agents_index.size, dtype='int32' ), new_agents_index ) self.processed_locations['add'] = concatenate((self.processed_locations.get('add', array([])), unique(agent_dataset[self.subarea_id_name][new_agents_index])))
def run(self, data, upc_sequence, resources=None): self.mnl_probabilities = upc_sequence.probability_class self.bhhh_estimation = bhhh_mnl_estimation() modified_upc_sequence = UPCFactory().get_model( utilities=None, probabilities="opus_core.mnl_probabilities", choices=None) modified_upc_sequence.utility_class = upc_sequence.utility_class N, neqs, V = data.shape max_iter = resources.get("max_iterations", 100) # default sc = SessionConfiguration() dataset_pool = sc.get_dataset_pool() sample_rate = dataset_pool.get_dataset("sample_rate") CLOSE = sc["CLOSE"] info_filename = sc["info_file"] info_filename = os.path.join('.', info_filename) info_file = open(info_filename, "a") constraint_dict = {1: 'constrained', 0: 'unconstrained'} swing_cases_fix = 0 #set swing alternatives to constrained (1) or unconstrained (0) prob_correlation = None choice_set = resources['_model_'].choice_set J = choice_set.size() alt_id = choice_set.get_id_attribute() movers = choice_set.get_attribute('movers') resources.check_obligatory_keys(["capacity_string"]) supply = choice_set.get_attribute(resources["capacity_string"]) index = resources.get("index", None) if index is None: # no sampling case, alternative set is the full choice_set index = arange(J) if index.ndim <= 1: index = repeat(index[newaxis, :], N, axis=0) if resources.get('aggregate_to_dataset', None): aggregate_dataset = dataset_pool.get_dataset( resources.get('aggregate_to_dataset')) choice_set_aggregate_id = choice_set.get_attribute( aggregate_dataset.get_id_name()[0]) index = aggregate_dataset.get_id_index( choice_set_aggregate_id[index].ravel()).reshape(index.shape) supply = aggregate_dataset.get_attribute( resources["capacity_string"]) J = aggregate_dataset.size() movers = aggregate_dataset.get_attribute("movers") demand_history = movers[:, newaxis] resources.merge({"index": index}) pi = ones(index.shape, dtype=float32) #initialize pi #average_omega = ones(J,dtype=float32) #initialize average_omega logger.start_block('Outer Loop') for i in range(max_iter): logger.log_status('Outer Loop Iteration %s' % i) result = self.bhhh_estimation.run(data, modified_upc_sequence, resources) del self.bhhh_estimation collect() self.bhhh_estimation = bhhh_mnl_estimation() probability = modified_upc_sequence.get_probabilities() if data.shape[2] == V: #insert a placeholder for ln(pi) in data data = concatenate((data, ones((N, neqs, 1), dtype=float32)), axis=2) coef_names = resources.get("coefficient_names") coef_names = concatenate((coef_names, array(["ln_pi"]))) resources.merge({"coefficient_names": coef_names}) else: beta_ln_pi = result['estimators'][where( coef_names == 'ln_pi')][0] logger.log_status("mu = 1/%s = %s" % (beta_ln_pi, 1 / beta_ln_pi)) prob_hat = safe_array_divide(probability, pi**beta_ln_pi) #prob_hat = safe_array_divide(probability, pi) prob_hat_sum = prob_hat.sum(axis=1, dtype=float32) if not ma.allclose(prob_hat_sum, 1.0): logger.log_status( "probability doesn't sum up to 1, with minimum %s, and maximum %s" % (prob_hat_sum.min(), prob_hat_sum.max())) probability = normalize(prob_hat) demand = self.mnl_probabilities.get_demand(index, probability, J) * 1 / sample_rate demand_history = concatenate((demand_history, demand[:, newaxis]), axis=1) sdratio = safe_array_divide( supply, demand, return_value_if_denominator_is_zero=2.0) sdratio_matrix = sdratio[index] ## debug info from numpy import histogram from opus_core.misc import unique cc = histogram(index.ravel(), unique(index.ravel()))[0] logger.log_status( "=================================================================" ) logger.log_status("Probability min: %s, max: %s" % (probability.min(), probability.max())) logger.log_status("Demand min: %s, max: %s" % (demand.min(), demand.max())) logger.log_status("sdratio min: %s, max: %s" % (sdratio.min(), sdratio.max())) logger.log_status("demand[sdratio==sdratio.min()]=%s" % demand[sdratio == sdratio.min()]) logger.log_status("demand[sdratio==sdratio.max()]=%s" % demand[sdratio == sdratio.max()]) logger.log_status( "Counts of unique submarkets in alternatives min: %s, max: %s" % (cc.min(), cc.max())) logger.log_status( "=================================================================" ) constrained_locations_matrix, omega, info = self.inner_loop( supply, demand, probability, index, sdratio_matrix, J, max_iteration=max_iter) inner_iterations, constrained_locations_history, swing_index, average_omega_history = info for idx in swing_index: logger.log_status( "swinging alt with id %s set to %s" % (alt_id[idx], constraint_dict[swing_cases_fix])) constrained_locations_matrix[index == idx] = swing_cases_fix if swing_index.size > 0: info_file.write("swing of constraints found with id %s \n" % alt_id[swing_index]) info_file.write("outer_iteration, %i, " % i + ", ".join([str(i)] * (len(inner_iterations))) + "\n") info_file.write("inner_iteration, , " + ", ".join(inner_iterations) + "\n") info_file.write("id, sdratio, " + ", ".join(["avg_omega"] * len(inner_iterations)) + "\n") for idx in swing_index: line = str(alt_id[idx]) + ',' line += str(sdratio[idx]) + ',' line += ",".join( [str(x) for x in average_omega_history[idx, ]]) line += "\n" info_file.write(line) info_file.write("\n") info_file.flush() outer_iterations = [str(i)] * len(inner_iterations) prob_min = [str(probability.min())] * len(inner_iterations) prob_max = [str(probability.max())] * len(inner_iterations) pi_new = self.mnl_probabilities.get_pi( sdratio_matrix, omega, constrained_locations_matrix) data[:, :, -1] = ln(pi_new) #diagnostic output if not ma.allclose(pi, pi_new, atol=CLOSE): if i > 0: #don't print this for the first iteration logger.log_status("min of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).min()) logger.log_status("max of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).max()) logger.log_status("mean of pi(l+1) - pi(l): %s" % (pi_new - pi).mean()) logger.log_status( 'Standard Deviation pi(l+1) - pi(l): %s' % standard_deviation(pi_new - pi)) logger.log_status('correlation of pi(l+1) and pi(l): %s' % corr(pi_new.ravel(), pi.ravel())[0, 1]) pi = pi_new probability_old = probability # keep probability of the previous loop, for statistics computation only else: #convergence criterion achieved, quiting outer loop logger.log_status( "pi(l) == pi(l+1): Convergence criterion achieved") info_file.write("\nConstrained Locations History:\n") info_file.write("outer_iteration," + ",".join(outer_iterations) + "\n") info_file.write("inner_iteration," + ",".join(inner_iterations) + "\n") info_file.write("minimum_probability," + ",".join(prob_min) + "\n") info_file.write("maximum_probability," + ",".join(prob_max) + "\n") for row in range(J): line = [ str(x) for x in constrained_locations_history[row, ] ] info_file.write( str(alt_id[row]) + "," + ",".join(line) + "\n") info_file.flush() info_file.write("\nDemand History:\n") i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", ,\n") for row in range(J): line = [str(x) for x in demand_history[row, ]] info_file.write( str(alt_id[row]) + "," + ",".join(line) + "\n") demand_history_info_criteria = [500, 100, 50, 20] for criterion in demand_history_info_criteria: com_rows_index = where(movers <= criterion)[0] info_file.write( "\nDemand History for alternatives with less than or equal to %s movers in 1998:\n" % criterion) i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", movers,\n") for row in com_rows_index: line = [str(x) for x in demand_history[row, ]] info_file.write( str(alt_id[row]) + "," + ",".join(line) + "\n") #import pdb; pdb.set_trace() #export prob correlation history correlation_indices, prob_correlation = self.compute_prob_correlation( probability_old, probability, prob_hat, index, resources) info_file.write("\nCorrelation of Probabilities:\n") c_name = [ 'corr(p_ij p~_ij)', 'corr(p_ij p^_ij)', 'corr(p_ij dummy)', 'corr(p~_ij p^_ij)', 'corr(p~_ij dummy)', 'corr(p^_ij dummy)' ] info_file.write("com_id, " + ",".join(c_name) + "\n") #info_file.write(", ,\n") for row in range(correlation_indices.size): line = [str(x) for x in prob_correlation[row, ]] info_file.write( str(alt_id[correlation_indices[row]]) + "," + ",".join(line) + "\n") info_file.close() result['pi'] = pi return result logger.end_block() try: info_file.close() except: pass raise RuntimeError, "max iteration reached without convergence."
def compute(self, dataset_pool): return safe_array_divide( self.get_dataset().get_attribute(self.building_units), self.get_dataset().get_attribute(self.number_of_units))
def compute(self, dataset_pool): return 100 * safe_array_divide( self.get_dataset().get_attribute(self.low_income), self.get_dataset().get_attribute(self.number_of_households), )
def compute(self, dataset_pool): ds = self.get_dataset() return safe_array_divide(ds.get_attribute(self.jobs), ds.get_attribute(self.acres_of_land))
def compute(self, dataset_pool): ds = self.get_dataset() return safe_array_divide(ds["number_of_jobs_of_sector_%s" % self.sector_id], ds["number_of_jobs"])
def compute(self, dataset_pool): acres = self.get_dataset().get_attribute(self.acres) return safe_array_divide(self.get_dataset().get_attribute(self.land_value),acres)
def compute(self, dataset_pool): nou = self.get_dataset().get_attribute("buildings_%s_space" % self.type) return safe_array_divide( self.get_dataset().get_attribute("total_value_%s" % self.type), nou)
def compute(self, dataset_pool): vl = self.get_dataset().get_attribute("vacant_land_sqft").astype("float32") return safe_array_divide( self.get_dataset().sum_dataset_over_ids(dataset_pool.get_dataset("gridcell"), "total_value_vacant_land"), vl )
def run(self, data, upc_sequence, resources=None): self.mnl_probabilities=upc_sequence.probability_class self.bhhh_estimation = bhhh_mnl_estimation() modified_upc_sequence = UPCFactory().get_model( utilities=None, probabilities="opus_core.mnl_probabilities", choices=None) modified_upc_sequence.utility_class = upc_sequence.utility_class N, neqs, V = data.shape max_iter = resources.get("max_iterations", 100) # default sc = SessionConfiguration() dataset_pool = sc.get_dataset_pool() sample_rate = dataset_pool.get_dataset("sample_rate") CLOSE = sc["CLOSE"] info_filename = sc["info_file"] info_filename = os.path.join('.', info_filename) info_file = open(info_filename, "a") constraint_dict = {1:'constrained', 0:'unconstrained'} swing_cases_fix = 0 #set swing alternatives to constrained (1) or unconstrained (0) prob_correlation = None choice_set = resources['_model_'].choice_set J = choice_set.size() alt_id = choice_set.get_id_attribute() movers = choice_set.get_attribute('movers') resources.check_obligatory_keys(["capacity_string"]) supply = choice_set.get_attribute(resources["capacity_string"]) index = resources.get("index", None) if index is None: # no sampling case, alternative set is the full choice_set index = arange(J) if index.ndim <= 1: index = repeat(index[newaxis,:], N, axis=0) if resources.get('aggregate_to_dataset', None): aggregate_dataset = dataset_pool.get_dataset(resources.get('aggregate_to_dataset')) choice_set_aggregate_id = choice_set.get_attribute(aggregate_dataset.get_id_name()[0]) index = aggregate_dataset.get_id_index(choice_set_aggregate_id[index].ravel()).reshape(index.shape) supply = aggregate_dataset.get_attribute(resources["capacity_string"]) J = aggregate_dataset.size() movers = aggregate_dataset.get_attribute("movers") demand_history = movers[:, newaxis] resources.merge({"index":index}) pi = ones(index.shape, dtype=float32) #initialize pi #average_omega = ones(J,dtype=float32) #initialize average_omega logger.start_block('Outer Loop') for i in range(max_iter): logger.log_status('Outer Loop Iteration %s' % i) result = self.bhhh_estimation.run(data, modified_upc_sequence, resources) del self.bhhh_estimation; collect() self.bhhh_estimation = bhhh_mnl_estimation() probability = modified_upc_sequence.get_probabilities() if data.shape[2] == V: #insert a placeholder for ln(pi) in data data = concatenate((data,ones((N,neqs,1),dtype=float32)), axis=2) coef_names = resources.get("coefficient_names") coef_names = concatenate( (coef_names, array(["ln_pi"])) ) resources.merge({"coefficient_names":coef_names}) else: beta_ln_pi = result['estimators'][where(coef_names == 'ln_pi')][0] logger.log_status("mu = 1/%s = %s" % (beta_ln_pi, 1/beta_ln_pi)) prob_hat = safe_array_divide(probability, pi ** beta_ln_pi) #prob_hat = safe_array_divide(probability, pi) prob_hat_sum = prob_hat.sum(axis=1, dtype=float32) if not ma.allclose(prob_hat_sum, 1.0): logger.log_status("probability doesn't sum up to 1, with minimum %s, and maximum %s" % (prob_hat_sum.min(), prob_hat_sum.max())) probability = normalize(prob_hat) demand = self.mnl_probabilities.get_demand(index, probability, J) * 1 / sample_rate demand_history = concatenate((demand_history, demand[:, newaxis]), axis=1) sdratio = safe_array_divide(supply, demand, return_value_if_denominator_is_zero=2.0) sdratio_matrix = sdratio[index] ## debug info from numpy import histogram from opus_core.misc import unique cc = histogram(index.ravel(), unique(index.ravel()))[0] logger.log_status( "=================================================================") logger.log_status( "Probability min: %s, max: %s" % (probability.min(), probability.max()) ) logger.log_status( "Demand min: %s, max: %s" % (demand.min(), demand.max()) ) logger.log_status( "sdratio min: %s, max: %s" % (sdratio.min(), sdratio.max()) ) logger.log_status( "demand[sdratio==sdratio.min()]=%s" % demand[sdratio==sdratio.min()] ) logger.log_status( "demand[sdratio==sdratio.max()]=%s" % demand[sdratio==sdratio.max()] ) logger.log_status( "Counts of unique submarkets in alternatives min: %s, max: %s" % (cc.min(), cc.max()) ) logger.log_status( "=================================================================") constrained_locations_matrix, omega, info = self.inner_loop(supply, demand, probability, index, sdratio_matrix, J, max_iteration=max_iter) inner_iterations, constrained_locations_history, swing_index, average_omega_history = info for idx in swing_index: logger.log_status("swinging alt with id %s set to %s" % (alt_id[idx], constraint_dict[swing_cases_fix])) constrained_locations_matrix[index==idx] = swing_cases_fix if swing_index.size > 0: info_file.write("swing of constraints found with id %s \n" % alt_id[swing_index]) info_file.write("outer_iteration, %i, " % i + ", ".join([str(i)]*(len(inner_iterations))) + "\n") info_file.write("inner_iteration, , " + ", ".join(inner_iterations) + "\n") info_file.write("id, sdratio, " + ", ".join(["avg_omega"]*len(inner_iterations)) + "\n") for idx in swing_index: line = str(alt_id[idx]) + ',' line += str(sdratio[idx]) + ',' line += ",".join([str(x) for x in average_omega_history[idx,]]) line += "\n" info_file.write(line) info_file.write("\n") info_file.flush() outer_iterations = [str(i)] * len(inner_iterations) prob_min = [str(probability.min())] * len(inner_iterations) prob_max = [str(probability.max())] * len(inner_iterations) pi_new = self.mnl_probabilities.get_pi(sdratio_matrix, omega, constrained_locations_matrix) data[:,:,-1] = ln(pi_new) #diagnostic output if not ma.allclose(pi, pi_new, atol=CLOSE): if i > 0: #don't print this for the first iteration logger.log_status("min of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).min()) logger.log_status("max of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).max()) logger.log_status("mean of pi(l+1) - pi(l): %s" % (pi_new - pi).mean()) logger.log_status('Standard Deviation pi(l+1) - pi(l): %s' % standard_deviation(pi_new - pi)) logger.log_status('correlation of pi(l+1) and pi(l): %s' % corr(pi_new.ravel(), pi.ravel())[0,1]) pi = pi_new probability_old = probability # keep probability of the previous loop, for statistics computation only else: #convergence criterion achieved, quiting outer loop logger.log_status("pi(l) == pi(l+1): Convergence criterion achieved") info_file.write("\nConstrained Locations History:\n") info_file.write("outer_iteration," + ",".join(outer_iterations) + "\n") info_file.write("inner_iteration," + ",".join(inner_iterations) + "\n") info_file.write("minimum_probability," + ",".join(prob_min) + "\n") info_file.write("maximum_probability," + ",".join(prob_max) + "\n") for row in range(J): line = [str(x) for x in constrained_locations_history[row,]] info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n") info_file.flush() info_file.write("\nDemand History:\n") i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", ,\n") for row in range(J): line = [str(x) for x in demand_history[row,]] info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n") demand_history_info_criteria = [500, 100, 50, 20] for criterion in demand_history_info_criteria: com_rows_index = where(movers <= criterion)[0] info_file.write("\nDemand History for alternatives with less than or equal to %s movers in 1998:\n" % criterion) i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", movers,\n") for row in com_rows_index: line = [str(x) for x in demand_history[row,]] info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n") #import pdb; pdb.set_trace() #export prob correlation history correlation_indices, prob_correlation = self.compute_prob_correlation(probability_old, probability, prob_hat, index, resources) info_file.write("\nCorrelation of Probabilities:\n") c_name = ['corr(p_ij p~_ij)', 'corr(p_ij p^_ij)', 'corr(p_ij dummy)', 'corr(p~_ij p^_ij)', 'corr(p~_ij dummy)', 'corr(p^_ij dummy)'] info_file.write("com_id, " + ",".join(c_name) + "\n") #info_file.write(", ,\n") for row in range(correlation_indices.size): line = [str(x) for x in prob_correlation[row,]] info_file.write(str(alt_id[correlation_indices[row]]) + "," + ",".join(line) + "\n") info_file.close() result['pi'] = pi return result logger.end_block() try:info_file.close() except:pass raise RuntimeError, "max iteration reached without convergence."
def _add(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool): fit_index = self.get_fit_agents_index( agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) movers_index = array([], dtype="int32") amount_from_agents_pool = min(amount, len(agents_pool)) if amount_from_agents_pool > 0: agents_index_from_agents_pool = sample_noreplace( agents_pool, amount_from_agents_pool) [agents_pool.remove(i) for i in agents_index_from_agents_pool] if fit_index.size == 0: ##cannot find agents to copy their location or clone them, place agents in agents_pool logger.log_warning("Refinement requests to add %i agents, but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \ "add %i agents instead" % (amount, amount_from_agents_pool, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression]).strip(' and '), amount_from_agents_pool,) ) amount = amount_from_agents_pool is_suitable_location = location_dataset.compute_variables( this_refinement.location_expression, dataset_pool=dataset_pool) location_id_for_agents_pool = sample_replace( location_dataset.get_id_attribute()[is_suitable_location], amount_from_agents_pool) else: agents_index_for_location = sample_replace( fit_index, amount_from_agents_pool) location_id_for_agents_pool = agent_dataset.get_attribute( location_dataset.get_id_name() [0])[agents_index_for_location] movers_index = concatenate( (movers_index, agents_index_for_location)) elif fit_index.size == 0: ## no agents in agents_pool and no agents to clone either, --> fail logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \ ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') ) return if amount > amount_from_agents_pool: agents_index_to_clone = sample_replace( fit_index, amount - amount_from_agents_pool) movers_index = concatenate((movers_index, agents_index_to_clone)) if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len( this_refinement.location_capacity_attribute) > 0: movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0])[movers_index] movers_location_index = location_dataset.get_id_index( movers_location_id) # see previous comment about histogram function num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() + 1))[0] num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \ ( location_dataset.dataset_name, agent_dataset.dataset_name ), dataset_pool=dataset_pool) expand_factor = safe_array_divide( (num_of_agents_by_location + num_of_movers_by_location).astype('float32'), num_of_agents_by_location, return_value_if_denominator_is_zero=1.0) new_values = round_(expand_factor * location_dataset.get_attribute( this_refinement.location_capacity_attribute)) location_dataset.modify_attribute( this_refinement.location_capacity_attribute, new_values) self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index) if amount_from_agents_pool > 0: agent_dataset.modify_attribute(location_dataset.get_id_name()[0], location_id_for_agents_pool, agents_index_from_agents_pool) self._add_refinement_info_to_dataset( agent_dataset, self.id_names, this_refinement, index=agents_index_from_agents_pool) if amount > amount_from_agents_pool: new_agents_index = agent_dataset.duplicate_rows( agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_to_clone) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=new_agents_index)