def compute(self, dataset_pool):
     vl = self.get_dataset().get_attribute("vacant_land_sqft").astype(
         "float32")
     return safe_array_divide(
         self.get_dataset().sum_dataset_over_ids(
             dataset_pool.get_dataset('gridcell'),
             "total_value_vacant_land"), vl)
 def compute(self,  dataset_pool):
     buildings = self.get_dataset()
     non_residential_sqft = buildings.get_attribute("non_residential_sqft")
     building_sqft_per_job = buildings.get_attribute("building_sqft_per_job")
     job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job'])
     job_spaces[buildings['year_built']<=2000] = buildings['job_capacity'][buildings['year_built']<=2000]
     return job_spaces
 def compute(self, dataset_pool):
     zone_set = self.get_dataset()
     travel_data = dataset_pool.get_dataset('travel_data')
     from_zone_id = travel_data.get_attribute('from_zone_id')
     zone_ids = zone_set.get_attribute('zone_id')
     time = travel_data.get_attribute(self.time_attribute_name)
     trips = travel_data.get_attribute(self.trips_attribute_name)
     
     numerator = array(ndimage_sum(time * trips,
                                    labels = from_zone_id, index=zone_ids))
     denominator = array(ndimage_sum(trips,
                                      labels = from_zone_id, index=zone_ids), dtype=float32)
     
     # if there is a divide by zero then subsititute the values from the zone one below that one
     # if there are contigious places of zero division the values should propigate upon iteration
     no_trips_from_here = where(denominator == 0)[0]
     while no_trips_from_here.size != 0:
         if no_trips_from_here.size == denominator.size:
             logger.log_warning("%s attribute of travel_data is all zeros; %s returns all zeros" % (self.trips_attribute_name, 
                                                                                                    self.name()
                                                                                                    ))
             break
              
         substitute_locations = no_trips_from_here - 1    # a mapping, what zone the new data will come from
         if substitute_locations[0] < 0: substitute_locations[0] = 1
         numerator[no_trips_from_here] = numerator[substitute_locations]
         denominator[no_trips_from_here] = denominator[substitute_locations] 
         no_trips_from_here = where(denominator == 0)[0]
         
     return safe_array_divide(numerator, denominator)
Exemple #4
0
    def compute(self,  dataset_pool):
        """ use networkx to determine whether two geographies are adjacent with
        k degree of expansion
        """

        zones = self.get_dataset()
        adjacent_zones = dataset_pool.get_dataset('adjacent_zone')
        id_max = zones['zone_id'].max()+1
        W = zeros((id_max, id_max), dtype='int8')  #diagonal
        
        sum_income = zeros(id_max, dtype=zones['sum_income'].dtype)
        sum_households = zeros(id_max, dtype=zones['sum_households'].dtype)
        sum_income[zones['zone_id']] = zones['sum_income']
        sum_households[zones['zone_id']] = zones['sum_households']
        if not nx: raise ImportError, "networkx module is required."
        G = nx.Graph()
        G.add_nodes_from(zones['zone_id'])
        G.add_edges_from(adjacent_zones.get_multiple_attributes(['zone_id', 'adjacent_zone_id']))
        length = nx.all_pairs_shortest_path_length(G, cutoff=self.order)
        
        for key, val in length.items():
            W[key][val.keys()] = 1
        
        sum_income = dot(W, sum_income[:, newaxis])[:, 0]
        sum_households = dot(W, sum_households[:, newaxis])[:, 0]

        results = safe_array_divide(sum_income, sum_households.astype('f'))
        return results[zones['zone_id']]
Exemple #5
0
 def compute_building_type_distribution(self, building_type_dataset, realestate_dataset_name, regions):
     parcels = self.dataset_pool.get_dataset('parcel')        
     building_type_ids = building_type_dataset.get_id_attribute()
     regions.compute_variables(map(lambda type:
         "units_proposed_for_bt_%s = %s.aggregate(psrc_parcel.parcel.units_proposed_for_building_type_%s)" % (type, self.subarea_name, type), 
         building_type_ids[logical_not(in1d(building_type_ids, self.bt_do_not_count))]),
                 dataset_pool=self.dataset_pool)
     sumunits_residential = zeros(regions.size(), dtype='float32')        
     sumunits_nonresidential = zeros(regions.size(), dtype='float32')
     for itype in range(building_type_ids.size):
         if building_type_ids[itype] in self.bt_do_not_count:
             continue
         potential_units = regions["units_proposed_for_bt_%s" % building_type_ids[itype]]
         if building_type_dataset['is_residential'][itype]:
             sumunits_residential = sumunits_residential + potential_units
         else:
             sumunits_nonresidential = sumunits_nonresidential + potential_units
     building_type_distribution = {}
     for itype in range(building_type_ids.size):
         building_type_distribution[itype] = zeros(regions.size())
         if building_type_ids[itype] in self.bt_do_not_count:
             continue
         if building_type_dataset['is_residential'][itype]:
             sumunits = sumunits_residential
         else:
             sumunits = sumunits_nonresidential
         building_type_distribution[itype] = safe_array_divide(regions['units_proposed_for_bt_%s' % building_type_ids[itype]], sumunits)         
     return building_type_distribution
Exemple #6
0
    def compute(self,  dataset_pool):
        zones = self.get_dataset()
        zone_ids = zones.get_id_attribute()
        travel_data = dataset_pool.get_dataset("travel_data")
#        import ipdb; ipdb.set_trace()
        if self.aggregate_by_origin:
            attribute_zone_id = travel_data.get_attribute("from_zone_id")
            summary_zone_id = travel_data.get_attribute("to_zone_id")
        else:
            attribute_zone_id = travel_data.get_attribute("to_zone_id")
            summary_zone_id = travel_data.get_attribute("from_zone_id")
            
        zone_index = zones.get_id_index(attribute_zone_id)
        attribute = zones[self.zone_attribute_to_access][zone_index]
        weight = safe_array_divide(1.0, 
                            power(travel_data[self.travel_data_attribute],2))
        
        function = getattr(ndimage, self.function)
        results = array(function(attribute * weight, 
                                 labels = summary_zone_id, 
                                 index=zone_ids))
        
        return results

## unittests in child class
    def _delete(self, agents_pool, amount, agent_dataset, location_dataset,
                this_refinement, dataset_pool):
        """similar to subtract action, instead of unplacing agents delete remove agents from the agent dataset,
        those agents won't be available for later action
        """

        fit_index = self.get_fit_agents_index(
            agent_dataset, this_refinement.agent_expression,
            this_refinement.location_expression, dataset_pool)

        if amount > fit_index.size or amount < 0:
            logger.log_warning("Refinement requests to delete %i agents,  but there are %i agents in total satisfying %s;" \
                               "delete %i agents instead" % (amount, fit_index.size,
                                                               ' and '.join( [this_refinement.agent_expression,
                                                                            this_refinement.location_expression] ).strip(' and '),
                                                               fit_index.size) )
            amount = fit_index.size

        if amount == fit_index.size:
            movers_index = fit_index
        else:
            movers_index = sample_noreplace(fit_index, amount)

        agents_pool = list(set(agents_pool) - set(movers_index))
        ## modify location capacity attribute if specified
        if this_refinement.location_capacity_attribute is not None and len(
                this_refinement.location_capacity_attribute) > 0:
            location_dataset = dataset_pool.get_dataset(
                VariableName(
                    this_refinement.location_expression).get_dataset_name())

            movers_location_id = agent_dataset.get_attribute(
                location_dataset.get_id_name()[0])[movers_index]
            movers_location_index = location_dataset.get_id_index(
                movers_location_id)
            # see previous comment about histogram function
            num_of_movers_by_location = histogram(
                movers_location_index,
                bins=arange(location_dataset.size() + 1))[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            (location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name),
                                                                            dataset_pool=dataset_pool)

            shrink_factor = safe_array_divide(
                (num_of_agents_by_location -
                 num_of_movers_by_location).astype('float32'),
                num_of_agents_by_location,
                return_value_if_denominator_is_zero=1.0)
            new_values = round_(shrink_factor * location_dataset.get_attribute(
                this_refinement.location_capacity_attribute))
            location_dataset.modify_attribute(
                this_refinement.location_capacity_attribute, new_values)
            self._add_refinement_info_to_dataset(location_dataset,
                                                 self.id_names,
                                                 this_refinement,
                                                 index=movers_location_index)

        agent_dataset.remove_elements(array(movers_index))
Exemple #8
0
 def compute(self,  dataset_pool):
     buildings = self.get_dataset()
     non_residential_sqft = buildings.get_attribute("non_residential_sqft")
     building_sqft_per_job = buildings.get_attribute("building_sqft_per_job")
     #job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job'])
     job_spaces = safe_array_divide(buildings['non_residential_sqft'], 250)
     ## only do this when job_capacity and year_built are primary attributes of buildings
     return job_spaces
Exemple #9
0
    def _subtract(self, agents_pool, amount, 
                  agent_dataset, location_dataset, 
                  this_refinement,
                  dataset_pool ):
        
        fit_index = self.get_fit_agents_index(agent_dataset, 
                                              this_refinement.agent_expression, 
                                              this_refinement.location_expression,
                                              dataset_pool)
        
        if amount > fit_index.size:
            logger.log_warning("Refinement requests to subtract %i agents,  but there are %i agents in total satisfying %s;" \
                               "subtract %i agents instead" % (amount, fit_index.size, 
                                                               ' and '.join( [this_refinement.agent_expression, 
                                                                            this_refinement.location_expression] ).strip(' and '),
                                                               fit_index.size) )
            amount = fit_index.size
        
        if amount == fit_index.size:
            movers_index = fit_index
        else:
            movers_index = sample_noreplace( fit_index, amount )
            
        agents_pool += movers_index.tolist()
        ## modify location capacity attribute if specified
        if this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0:
            location_dataset = dataset_pool.get_dataset( VariableName( this_refinement.location_expression ).get_dataset_name() )

            movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index]
            movers_location_index = location_dataset.get_id_index( movers_location_id )
            # backward compatability code for older versions of numpy -- no longer required since we need numpy 1.2.1 or greater
            # new=False argument to histogram tells it to use deprecated behavior for now (to be removed in numpy 1.3)
            # See numpy release notes -- search for histogram
            # if numpy.__version__ >= '1.2.0':
            #    num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()), new=False)[0]
            # else:
            #    num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()))[0]
            num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            (location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name),
                                                                            dataset_pool=dataset_pool)
            
            shrink_factor = safe_array_divide( (num_of_agents_by_location - num_of_movers_by_location ).astype('float32'),
                                                num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0  )
            new_values = round_( shrink_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) )
            location_dataset.modify_attribute( this_refinement.location_capacity_attribute, 
                                               new_values
                                               )
            self._add_refinement_info_to_dataset(location_dataset, ("refinement_id", "transaction_id"), this_refinement, index=movers_location_index)
            
        agent_dataset.modify_attribute(location_dataset.get_id_name()[0], 
                                       -1 * ones( movers_index.size, dtype='int32' ),
                                       index = movers_index
                                       )
        self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=movers_index)
Exemple #10
0
    def compute(self, dataset_pool):
        submarket = self.get_dataset()
        uptake = clip(submarket['households'] - submarket['households_lag1'], 0, inf) 
        vacant_units = clip(submarket['residential_units'] - submarket['households'], 0, inf)
        results = safe_array_divide( uptake, vacant_units.astype('f') )
        results = clip(results, self.lower_bound, self.upper_bound)        
        all_units_occupied = logical_and(vacant_units==0, submarket['households'] > 0)
        results[all_units_occupied] = self.upper_bound

        return results
 def compute(self, dataset_pool):
     buildings = self.get_dataset()
     non_residential_sqft = buildings.get_attribute("non_residential_sqft")
     building_sqft_per_job = buildings.get_attribute(
         "building_sqft_per_job")
     job_spaces = safe_array_divide(buildings['non_residential_sqft'],
                                    buildings['building_sqft_per_job'])
     job_spaces[buildings['year_built'] <= 2000] = buildings[
         'job_capacity'][buildings['year_built'] <= 2000]
     return job_spaces
Exemple #12
0
    def compute_sdratio(self, submarkets, choice_set, capacity_string, demand_string):
        #choice_submarket_id = choice_set.compute_variables("urbansim_parcel.%s.%s" % (choice_set.get_dataset_name(), submarkets.get_id_name()[0]))
        choice_submarket_id = choice_set.get_attribute("building_id")
        choice_submarket_index = submarkets.get_id_index( choice_submarket_id )
        supply = submarkets.sum_over_ids( choice_submarket_id, self.capacity )

        demand = submarkets.compute_variables("demand=submarket.aggregate(%s.%s)" % (choice_set.get_dataset_name(), demand_string) )
        ##TODO: an adjustment ratio should be included since this variable only represents the demand from agents in this chunk
        sdratio_submarket = safe_array_divide(supply, demand)

        return (sdratio_submarket, sdratio_submarket[choice_submarket_index] )
 def compute(self, dataset_pool):
     nb = self.get_dataset().get_attribute(self.number_of_buildings)
     buildings = dataset_pool.get_dataset('building')
     age = buildings.get_attribute(self.building_age)
     mask = ma.getmask(age)
     if mask is ma.nomask:
         donotcount=0
     else:
     # count number of buildings that have missing year_built and therefore won't be included in the average.
         donotcount = self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"), mask.astype("int8"))
     # fill -1 for masked values, in order to distinguish it from age=0.
     return safe_array_divide(self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"),
                                   age), nb-donotcount, -1)
Exemple #14
0
    def _delete(self, agents_pool, amount, 
                  agent_dataset, location_dataset, 
                  this_refinement,
                  dataset_pool ):
        """similar to subtract action, instead of unplacing agents delete remove agents from the agent dataset,
        those agents won't be available for later action
        """
        
        fit_index = self.get_fit_agents_index(agent_dataset, 
                                              this_refinement.agent_expression, 
                                              this_refinement.location_expression,
                                              dataset_pool)
        
        if amount > fit_index.size or amount < 0:
            logger.log_warning("Refinement requests to delete %i agents,  but there are %i agents in total satisfying %s;" \
                               "delete %i agents instead" % (amount, fit_index.size, 
                                                               ' and '.join( [this_refinement.agent_expression, 
                                                                            this_refinement.location_expression] ).strip(' and '),
                                                               fit_index.size) )
            amount = fit_index.size
        
        if amount == fit_index.size:
            movers_index = fit_index
        else:
            movers_index = sample_noreplace( fit_index, amount )
            
        agents_pool = list( set(agents_pool) - set(movers_index) )
        ## modify location capacity attribute if specified
        if this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0:
            location_dataset = dataset_pool.get_dataset( VariableName( this_refinement.location_expression ).get_dataset_name() )

            movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index]
            movers_location_index = location_dataset.get_id_index( movers_location_id )
            # see previous comment about histogram function
            num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            (location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name),
                                                                            dataset_pool=dataset_pool)
            
            shrink_factor = safe_array_divide( (num_of_agents_by_location - num_of_movers_by_location ).astype('float32'),
                                                num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0  )
            new_values = round_( shrink_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) )
            location_dataset.modify_attribute( this_refinement.location_capacity_attribute, 
                                               new_values
                                               )
            self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index)
            
        agent_dataset.remove_elements( array(movers_index) )
Exemple #15
0
 def compute_job_building_type_distribution(self, building_type_dataset, realestate_dataset_name, regions):
     building_type_ids = building_type_dataset.get_id_attribute()
     
     regions.compute_variables(map(lambda type: 
         "number_of_jobs_for_bt_%s = %s.aggregate(psrc_parcel.%s.number_of_non_home_based_jobs * (%s.building_type_id == %s))" % (type, self.subarea_name, realestate_dataset_name, realestate_dataset_name, type),
             building_type_ids[logical_not(in1d(building_type_ids, self.bt_do_not_count))]) + 
           ["number_of_nhb_jobs = %s.aggregate(psrc_parcel.%s.number_of_non_home_based_jobs)" % (self.subarea_name, realestate_dataset_name)], 
                               dataset_pool=self.dataset_pool)
     building_type_distribution = {}
     for itype in range(building_type_ids.size):
         building_type_distribution[itype] = zeros(regions.size())
         if building_type_ids[itype] in self.bt_do_not_count:
             continue            
         building_type_distribution[itype] = safe_array_divide(
                 regions['number_of_jobs_for_bt_%s' % building_type_ids[itype]], regions['number_of_nhb_jobs'])
     return building_type_distribution
Exemple #16
0
 def compute(self,  dataset_pool):
     buildings = self.get_dataset()
     non_residential_sqft = buildings.get_attribute("non_residential_sqft")
     building_sqft_per_job = buildings.get_attribute("building_sqft_per_job")
     job_spaces = safe_array_divide(buildings['non_residential_sqft'], buildings['building_sqft_per_job'])
     ## only do this when job_capacity and year_built are primary attributes of buildings
     known_names = buildings.get_known_attribute_names()
     if 'job_capacity' in known_names: # and 'year_built' in known_names:
         self.add_and_solve_dependencies(["building.job_capacity"#, "building.year_built"
                                          ], dataset_pool=dataset_pool)
         #base_year = SimulationState().get_start_time()
         #base_year = 2000 # This can't be taken from SimulationState because of running refinement with different base year
         #job_spaces[buildings['year_built']<=base_year] = buildings['job_capacity'][buildings['year_built']<=base_year]
         # Rather than using year_built we can do the following because the building construction model assigns -1 to new buildings
         job_spaces[buildings['job_capacity'] >= 0] = buildings['job_capacity'][buildings['job_capacity'] >= 0] 
     return job_spaces
    def compute(self, dataset_pool):
        nb = self.get_dataset().get_attribute(self.number_of_buildings)
        buildings = dataset_pool.get_dataset('building')
        age = buildings.get_attribute(
            self.building_age) * buildings.get_attribute(self.is_building_type)
        mask = ma.getmask(age)
        if mask is ma.nomask:
            donotcount = 0
        else:
            mask = mask * buildings.get_attribute(self.is_building_type)
            # count number of buildings that have missing year_built and therefore won't be included in the average.
            donotcount = self.get_dataset().sum_over_ids(
                buildings.get_attribute("grid_id"), mask.astype("int8"))

        # fill -1 for masked values, in order to distinguish it from age=0.
        return safe_array_divide(
            self.get_dataset().sum_over_ids(buildings.get_attribute("grid_id"),
                                            age), nb - donotcount, -1)
    def compute(self, dataset_pool):
        proposals = self.get_dataset()
        
        unit_price = proposals.get_attribute("unit_price")
        existing_units = proposals.get_attribute("existing_units")
        improvement_value = proposals.get_attribute("improvement_value")
        land_area_taken = proposals.get_attribute("land_area_taken")
        land_area = proposals.get_attribute("land_area")
        
        ## is_redevelopment is optional, if is_redevelopment is not one of proposal attributes, default to 0
        if 'is_redevelopment' in proposals.get_known_attribute_names():
            is_redevelopment = proposals.get_attribute('is_redevelopment')
        else:
            is_redevelopment = 0
        
        results = safe_array_divide( (unit_price * existing_units - improvement_value), land_area) * land_area_taken + improvement_value * is_redevelopment

        return results
    def compute(self, dataset_pool):
        proposals = self.get_dataset()

        unit_price = proposals.get_attribute("unit_price")
        existing_units = proposals.get_attribute("existing_units")
        improvement_value = proposals.get_attribute("improvement_value")
        land_area_taken = proposals.get_attribute("land_area_taken")
        land_area = proposals.get_attribute("land_area")

        ## is_redevelopment is optional, if is_redevelopment is not one of proposal attributes, default to 0
        if 'is_redevelopment' in proposals.get_known_attribute_names():
            is_redevelopment = proposals.get_attribute('is_redevelopment')
        else:
            is_redevelopment = 0

        results = safe_array_divide(
            (unit_price * existing_units - improvement_value),
            land_area) * land_area_taken + improvement_value * is_redevelopment

        return results
Exemple #20
0
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     return safe_array_divide(ds.get_attribute(self.vacant_units), 
                              ds.get_attribute(self.number_of_units))
Exemple #21
0
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     alldata = dataset_pool.get_dataset('alldata')
     results = ds['M_{0}j0'.format(self.type_id)] * safe_array_divide(alldata['TM_{0}t'.format(self.type_id)], alldata['TM_{0}0'.format(self.type_id)]) * safe_array_divide(ds['V_ijt'], ds['V_ij0'])
     return results
Exemple #22
0
 def compute(self, dataset_pool):
     nh = self.get_dataset().get_attribute(self.number_of_households)
     return safe_array_divide(
         self.get_dataset().get_attribute(self.population), nh)
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     values_sqft_per_job = ds.get_attribute(self.sqft_per_job)
     values_sqft = ds.get_attribute(self.sqft)
     return safe_array_divide(values_sqft, values_sqft_per_job, type="int32")
 def compute(self, dataset_pool):
     sp = self.get_dataset().get_attribute("%s_space" % self.type)
     return safe_array_divide(self.get_dataset().get_attribute("total_value_%s" % self.type), sp)
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     return safe_array_divide(ds.get_attribute(self.vacant_units),
                              ds.get_attribute(self.number_of_units))
 def compute(self, dataset_pool):
     acres = self.get_dataset().get_attribute(self.acres)
     return safe_array_divide(
         self.get_dataset().get_attribute(self.land_value), acres)
 def compute(self, dataset_pool):
     nou = self.get_dataset().get_attribute("buildings_%s_space_where_total_value" % self.type)
     return safe_array_divide(self.get_dataset().get_attribute(
             "total_value_%s" % self.type), nou)
Exemple #28
0
 def compute(self, dataset_pool):
     return 100 * safe_array_divide(
         self.get_dataset().get_attribute(self.low_income),
         self.get_dataset().get_attribute(self.number_of_households))
Exemple #29
0
    def _add(self, agents_pool, amount, 
             agent_dataset, location_dataset, 
             this_refinement,
             dataset_pool ):
        
        fit_index = self.get_fit_agents_index(agent_dataset, 
                                              this_refinement.agent_expression, 
                                              this_refinement.location_expression,
                                              dataset_pool)
        movers_index = array([],dtype="int32")
        amount_from_agents_pool = min( amount, len(agents_pool) )
        if amount_from_agents_pool > 0:
            agents_index_from_agents_pool = sample_noreplace( agents_pool, amount_from_agents_pool )
            [ agents_pool.remove(i) for i in agents_index_from_agents_pool ]
            if fit_index.size == 0:
                ##cannot find agents to copy their location or clone them, place agents in agents_pool
                if amount > amount_from_agents_pool:                   
                    logger.log_warning("Refinement requests to add %i agents,  but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \
                                   "add %i agents instead" % (amount, amount_from_agents_pool, 
                                                              ' and '.join( [this_refinement.agent_expression, 
                                                                           this_refinement.location_expression]).strip(' and '), 
                                                              amount_from_agents_pool,) )
                    amount = amount_from_agents_pool
                # sample from all suitable locations
                is_suitable_location = location_dataset.compute_variables( this_refinement.location_expression,
                                                                           dataset_pool=dataset_pool )
                location_id_for_agents_pool = sample_replace( location_dataset.get_id_attribute()[is_suitable_location],
                                                                 amount_from_agents_pool )
            else:
                #sample from locations of suitable agents            
                agents_index_for_location = sample_replace( fit_index, amount_from_agents_pool)
                location_id_for_agents_pool = agent_dataset.get_attribute( location_dataset.get_id_name()[0] 
                                                                         )[agents_index_for_location]
                movers_index = concatenate( (movers_index, agents_index_for_location) )

        elif fit_index.size == 0:
            ## no agents in agents_pool and no agents to clone either, --> fail
            logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \
                              ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') )
            return
            
        if amount > amount_from_agents_pool:
            agents_index_to_clone = sample_replace( fit_index, amount - amount_from_agents_pool)
            movers_index = concatenate( (movers_index, agents_index_to_clone) )

        if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0:
            movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index]
            movers_location_index = location_dataset.get_id_index( movers_location_id )
            # see previous comment about histogram function
            num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            ( location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name ),
                                                                            dataset_pool=dataset_pool)
            
            expand_factor = safe_array_divide( (num_of_agents_by_location + num_of_movers_by_location ).astype('float32'),
                                                num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 )
            new_values = round_( expand_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) )
            location_dataset.modify_attribute( this_refinement.location_capacity_attribute, 
                                               new_values
                                           )
            self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index)
        if amount_from_agents_pool > 0:
            agent_dataset.modify_attribute( location_dataset.get_id_name()[0],
                                            location_id_for_agents_pool,
                                            agents_index_from_agents_pool
                                            )
            self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_from_agents_pool)
        if amount > amount_from_agents_pool:
            new_agents_index = agent_dataset.duplicate_rows(agents_index_to_clone)
            self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_to_clone)
            self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=new_agents_index)
    def _subtract(self, agents_pool, amount, agent_dataset, location_dataset,
                  this_refinement, dataset_pool):

        fit_index = self.get_fit_agents_index(
            agent_dataset, this_refinement.agent_expression,
            this_refinement.location_expression, dataset_pool)

        if amount > fit_index.size:
            logger.log_warning("Refinement requests to subtract %i agents,  but there are %i agents in total satisfying %s;" \
                               "subtract %i agents instead" % (amount, fit_index.size,
                                                               ' and '.join( [this_refinement.agent_expression,
                                                                            this_refinement.location_expression] ).strip(' and '),
                                                               fit_index.size) )
            amount = fit_index.size

        if amount == fit_index.size:
            movers_index = fit_index
        else:
            movers_index = sample_noreplace(fit_index, amount)

        agents_pool += movers_index.tolist()
        ## modify location capacity attribute if specified
        if this_refinement.location_capacity_attribute is not None and len(
                this_refinement.location_capacity_attribute) > 0:
            location_dataset = dataset_pool.get_dataset(
                VariableName(
                    this_refinement.location_expression).get_dataset_name())

            movers_location_id = agent_dataset.get_attribute(
                location_dataset.get_id_name()[0])[movers_index]
            movers_location_index = location_dataset.get_id_index(
                movers_location_id)
            # backward compatability code for older versions of numpy -- no longer required since we need numpy 1.2.1 or greater
            # new=False argument to histogram tells it to use deprecated behavior for now (to be removed in numpy 1.3)
            # See numpy release notes -- search for histogram
            # if numpy.__version__ >= '1.2.0':
            #    num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()), new=False)[0]
            # else:
            #    num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size()))[0]
            num_of_movers_by_location = histogram(
                movers_location_index,
                bins=arange(location_dataset.size() + 1))[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            (location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name),
                                                                            dataset_pool=dataset_pool)

            shrink_factor = safe_array_divide(
                (num_of_agents_by_location -
                 num_of_movers_by_location).astype('float32'),
                num_of_agents_by_location,
                return_value_if_denominator_is_zero=1.0)
            new_values = round_(shrink_factor * location_dataset.get_attribute(
                this_refinement.location_capacity_attribute))
            location_dataset.modify_attribute(
                this_refinement.location_capacity_attribute, new_values)
            self._add_refinement_info_to_dataset(
                location_dataset, ("refinement_id", "transaction_id"),
                this_refinement,
                index=movers_location_index)

        agent_dataset.modify_attribute(location_dataset.get_id_name()[0],
                                       -1 *
                                       ones(movers_index.size, dtype='int32'),
                                       index=movers_index)
        self._add_refinement_info_to_dataset(agent_dataset,
                                             self.id_names,
                                             this_refinement,
                                             index=movers_index)
 def compute(self, dataset_pool):
     return safe_array_divide(self.get_dataset().get_attribute(self.building_units),
             self.get_dataset().get_attribute(self.number_of_units))
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     return safe_array_divide(ds.get_attribute(self.population),
                              ds.get_attribute(self.acres_of_land))
 def compute(self, dataset_pool):
     nh = self.get_dataset().get_attribute(self.number_of_households)
     return safe_array_divide(self.get_dataset().get_attribute(self.population), nh)
Exemple #34
0
    def _add(self, agents_pool, amount, 
             agent_dataset, location_dataset, 
             this_refinement,
             dataset_pool ):
        
        fit_index = self.get_fit_agents_index(agent_dataset, 
                                              this_refinement.agent_expression, 
                                              this_refinement.location_expression,
                                              dataset_pool)
        if this_refinement.agent_expression is not None and len(this_refinement.agent_expression) > 0:
            agents_index = where(agent_dataset.compute_variables(this_refinement.agent_expression, 
                                                               dataset_pool=dataset_pool)>0)[0]
        else:
            agents_index = arange(agent_dataset.size())
        movers_index = array([],dtype="int32")
        ar_pool = array(agents_pool)
        fitted_agents_pool = ar_pool[in1d(ar_pool, agents_index)]
        amount_from_agents_pool = min( amount, fitted_agents_pool.size )
        prob_string = self.probability_attributes.get(agent_dataset.get_dataset_name(),None)
        if prob_string is not None:
            probs_values = (agent_dataset.compute_variables([prob_string], dataset_pool=dataset_pool)).astype('int32')
            uprobs_values = unique(probs_values[fit_index])
            if uprobs_values.size > 0:
                probs_existing = array(ndimage_sum(ones(fit_index.size), 
                                         labels=probs_values[fit_index], index=uprobs_values))
        if amount_from_agents_pool > 0:        
            if prob_string is not None and uprobs_values.size > 0:                
                prob_pool_values = probs_values[fitted_agents_pool]
                probs_pool=zeros(prob_pool_values.size)
                for i in range(uprobs_values.size):
                    probpoolidx = where(prob_pool_values == uprobs_values[i])[0]
                    if probpoolidx.size == 0:
                        continue
                    probs_pool[probpoolidx]=probs_existing[i]/float(probpoolidx.size)
                probs_pool[probs_pool<=0] = (probs_existing.min()/10.0)/float((probs_pool<=0).sum())
            else:
                probs_pool=ones(fitted_agents_pool.size)
            
            agents_index_from_agents_pool = probsample_noreplace( fitted_agents_pool, amount_from_agents_pool, prob_array=probs_pool )
            [ agents_pool.remove(i) for i in agents_index_from_agents_pool ]
            if fit_index.size == 0:
                ##cannot find agents to copy their location or clone them, place agents in agents_pool
                if amount > amount_from_agents_pool:                   
                    logger.log_warning("Refinement requests to add %i agents,  but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \
                                   "add %i agents instead" % (amount, amount_from_agents_pool, 
                                                              ' and '.join( [this_refinement.agent_expression, 
                                                                           this_refinement.location_expression]).strip(' and '), 
                                                              amount_from_agents_pool,) )
                    amount = amount_from_agents_pool
                # sample from all suitable locations
                is_suitable_location = location_dataset.compute_variables( this_refinement.location_expression,
                                                                           dataset_pool=dataset_pool )
                location_id_for_agents_pool = sample_replace( location_dataset.get_id_attribute()[is_suitable_location],
                                                                 amount_from_agents_pool )
            else:
                #sample from locations of suitable agents            
                agents_index_for_location = sample_replace( fit_index, amount_from_agents_pool)
                location_id_for_agents_pool = agent_dataset.get_attribute( location_dataset.get_id_name()[0] 
                                                                         )[agents_index_for_location]
                movers_index = concatenate( (movers_index, agents_index_for_location) )

        elif fit_index.size == 0:
            ## no agents in agents_pool and no agents to clone either, --> fail
            logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \
                              ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') )
            return
            
        if amount > amount_from_agents_pool:
            agents_index_to_clone = sample_replace( fit_index, amount - amount_from_agents_pool)
            movers_index = concatenate( (movers_index, agents_index_to_clone) )

        if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len(this_refinement.location_capacity_attribute) > 0:
            movers_location_id = agent_dataset.get_attribute( location_dataset.get_id_name()[0] )[movers_index]
            movers_location_index = location_dataset.get_id_index( movers_location_id )
            # see previous comment about histogram function
            num_of_movers_by_location = histogram( movers_location_index, bins=arange(location_dataset.size() +1) )[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            ( location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name ),
                                                                            dataset_pool=dataset_pool)
            
            expand_factor = safe_array_divide( (num_of_agents_by_location + num_of_movers_by_location ).astype('float32'),
                                                num_of_agents_by_location, return_value_if_denominator_is_zero = 1.0 )
            new_values = round_( expand_factor * location_dataset.get_attribute(this_refinement.location_capacity_attribute) )
            location_dataset.modify_attribute( this_refinement.location_capacity_attribute, 
                                               new_values
                                           )
            self._add_refinement_info_to_dataset(location_dataset, self.id_names, this_refinement, index=movers_location_index)
        if amount_from_agents_pool > 0:
            agent_dataset.modify_attribute( 'building_id',
                                            -1 * ones( agents_index_from_agents_pool.size, dtype='int32' ),
                                            agents_index_from_agents_pool
                                            )
            agent_dataset.modify_attribute( location_dataset.get_id_name()[0],
                                            location_id_for_agents_pool,
                                            agents_index_from_agents_pool
                                            )

            self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_from_agents_pool)
            self.processed_locations['add'] = concatenate((self.processed_locations.get('add', array([])), 
                                                unique(location_dataset[self.subarea_id_name][location_dataset.get_id_index(location_id_for_agents_pool)])))
            
        if amount > amount_from_agents_pool:
            new_agents_index = agent_dataset.duplicate_rows(agents_index_to_clone)
            self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=agents_index_to_clone)
            self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=new_agents_index)
            if location_dataset.get_dataset_name() <> 'building':
                agent_dataset.modify_attribute( 'building_id',
                                            -1 * ones( new_agents_index.size, dtype='int32' ),
                                            new_agents_index
                                            )
            self.processed_locations['add'] = concatenate((self.processed_locations.get('add', array([])), 
                                                unique(agent_dataset[self.subarea_id_name][new_agents_index])))
    def run(self, data, upc_sequence, resources=None):

        self.mnl_probabilities = upc_sequence.probability_class
        self.bhhh_estimation = bhhh_mnl_estimation()

        modified_upc_sequence = UPCFactory().get_model(
            utilities=None,
            probabilities="opus_core.mnl_probabilities",
            choices=None)
        modified_upc_sequence.utility_class = upc_sequence.utility_class

        N, neqs, V = data.shape

        max_iter = resources.get("max_iterations", 100)  # default
        sc = SessionConfiguration()
        dataset_pool = sc.get_dataset_pool()
        sample_rate = dataset_pool.get_dataset("sample_rate")

        CLOSE = sc["CLOSE"]
        info_filename = sc["info_file"]
        info_filename = os.path.join('.', info_filename)
        info_file = open(info_filename, "a")
        constraint_dict = {1: 'constrained', 0: 'unconstrained'}
        swing_cases_fix = 0  #set swing alternatives to constrained (1) or unconstrained (0)
        prob_correlation = None

        choice_set = resources['_model_'].choice_set
        J = choice_set.size()
        alt_id = choice_set.get_id_attribute()
        movers = choice_set.get_attribute('movers')

        resources.check_obligatory_keys(["capacity_string"])
        supply = choice_set.get_attribute(resources["capacity_string"])

        index = resources.get("index", None)
        if index is None:  # no sampling case, alternative set is the full choice_set
            index = arange(J)
        if index.ndim <= 1:
            index = repeat(index[newaxis, :], N, axis=0)

        if resources.get('aggregate_to_dataset', None):
            aggregate_dataset = dataset_pool.get_dataset(
                resources.get('aggregate_to_dataset'))
            choice_set_aggregate_id = choice_set.get_attribute(
                aggregate_dataset.get_id_name()[0])
            index = aggregate_dataset.get_id_index(
                choice_set_aggregate_id[index].ravel()).reshape(index.shape)

            supply = aggregate_dataset.get_attribute(
                resources["capacity_string"])
            J = aggregate_dataset.size()

            movers = aggregate_dataset.get_attribute("movers")

        demand_history = movers[:, newaxis]
        resources.merge({"index": index})

        pi = ones(index.shape, dtype=float32)  #initialize pi
        #average_omega = ones(J,dtype=float32)  #initialize average_omega
        logger.start_block('Outer Loop')
        for i in range(max_iter):
            logger.log_status('Outer Loop Iteration %s' % i)

            result = self.bhhh_estimation.run(data, modified_upc_sequence,
                                              resources)
            del self.bhhh_estimation
            collect()
            self.bhhh_estimation = bhhh_mnl_estimation()

            probability = modified_upc_sequence.get_probabilities()
            if data.shape[2] == V:  #insert a placeholder for ln(pi) in data
                data = concatenate((data, ones((N, neqs, 1), dtype=float32)),
                                   axis=2)
                coef_names = resources.get("coefficient_names")
                coef_names = concatenate((coef_names, array(["ln_pi"])))
                resources.merge({"coefficient_names": coef_names})
            else:
                beta_ln_pi = result['estimators'][where(
                    coef_names == 'ln_pi')][0]
                logger.log_status("mu = 1/%s = %s" %
                                  (beta_ln_pi, 1 / beta_ln_pi))

                prob_hat = safe_array_divide(probability, pi**beta_ln_pi)
                #prob_hat = safe_array_divide(probability, pi)
                prob_hat_sum = prob_hat.sum(axis=1, dtype=float32)
                if not ma.allclose(prob_hat_sum, 1.0):
                    logger.log_status(
                        "probability doesn't sum up to 1, with minimum %s, and maximum %s"
                        % (prob_hat_sum.min(), prob_hat_sum.max()))

                    probability = normalize(prob_hat)

            demand = self.mnl_probabilities.get_demand(index, probability,
                                                       J) * 1 / sample_rate
            demand_history = concatenate((demand_history, demand[:, newaxis]),
                                         axis=1)

            sdratio = safe_array_divide(
                supply, demand, return_value_if_denominator_is_zero=2.0)
            sdratio_matrix = sdratio[index]
            ## debug info
            from numpy import histogram
            from opus_core.misc import unique
            cc = histogram(index.ravel(), unique(index.ravel()))[0]
            logger.log_status(
                "================================================================="
            )
            logger.log_status("Probability min: %s, max: %s" %
                              (probability.min(), probability.max()))
            logger.log_status("Demand min: %s, max: %s" %
                              (demand.min(), demand.max()))
            logger.log_status("sdratio min: %s, max: %s" %
                              (sdratio.min(), sdratio.max()))
            logger.log_status("demand[sdratio==sdratio.min()]=%s" %
                              demand[sdratio == sdratio.min()])
            logger.log_status("demand[sdratio==sdratio.max()]=%s" %
                              demand[sdratio == sdratio.max()])
            logger.log_status(
                "Counts of unique submarkets in alternatives min: %s, max: %s"
                % (cc.min(), cc.max()))
            logger.log_status(
                "================================================================="
            )

            constrained_locations_matrix, omega, info = self.inner_loop(
                supply,
                demand,
                probability,
                index,
                sdratio_matrix,
                J,
                max_iteration=max_iter)

            inner_iterations, constrained_locations_history, swing_index, average_omega_history = info

            for idx in swing_index:
                logger.log_status(
                    "swinging alt with id %s set to %s" %
                    (alt_id[idx], constraint_dict[swing_cases_fix]))
                constrained_locations_matrix[index == idx] = swing_cases_fix

            if swing_index.size > 0:
                info_file.write("swing of constraints found with id %s \n" %
                                alt_id[swing_index])
                info_file.write("outer_iteration, %i, " % i +
                                ", ".join([str(i)] *
                                          (len(inner_iterations))) + "\n")
                info_file.write("inner_iteration, , " +
                                ", ".join(inner_iterations) + "\n")
                info_file.write("id, sdratio, " +
                                ", ".join(["avg_omega"] *
                                          len(inner_iterations)) + "\n")
                for idx in swing_index:
                    line = str(alt_id[idx]) + ','
                    line += str(sdratio[idx]) + ','
                    line += ",".join(
                        [str(x) for x in average_omega_history[idx, ]])
                    line += "\n"
                    info_file.write(line)

                info_file.write("\n")
                info_file.flush()

            outer_iterations = [str(i)] * len(inner_iterations)
            prob_min = [str(probability.min())] * len(inner_iterations)
            prob_max = [str(probability.max())] * len(inner_iterations)

            pi_new = self.mnl_probabilities.get_pi(
                sdratio_matrix, omega, constrained_locations_matrix)

            data[:, :, -1] = ln(pi_new)
            #diagnostic output

            if not ma.allclose(pi, pi_new, atol=CLOSE):
                if i > 0:  #don't print this for the first iteration
                    logger.log_status("min of abs(pi(l+1) - pi(l)): %s" %
                                      absolute(pi_new - pi).min())
                    logger.log_status("max of abs(pi(l+1) - pi(l)): %s" %
                                      absolute(pi_new - pi).max())
                    logger.log_status("mean of pi(l+1) - pi(l): %s" %
                                      (pi_new - pi).mean())
                    logger.log_status(
                        'Standard Deviation pi(l+1) - pi(l): %s' %
                        standard_deviation(pi_new - pi))
                    logger.log_status('correlation of pi(l+1) and pi(l): %s' %
                                      corr(pi_new.ravel(), pi.ravel())[0, 1])

                pi = pi_new
                probability_old = probability  # keep probability of the previous loop, for statistics computation only
            else:  #convergence criterion achieved, quiting outer loop
                logger.log_status(
                    "pi(l) == pi(l+1): Convergence criterion achieved")

                info_file.write("\nConstrained Locations History:\n")
                info_file.write("outer_iteration," +
                                ",".join(outer_iterations) + "\n")
                info_file.write("inner_iteration," +
                                ",".join(inner_iterations) + "\n")
                info_file.write("minimum_probability," + ",".join(prob_min) +
                                "\n")
                info_file.write("maximum_probability," + ",".join(prob_max) +
                                "\n")
                for row in range(J):
                    line = [
                        str(x) for x in constrained_locations_history[row, ]
                    ]
                    info_file.write(
                        str(alt_id[row]) + "," + ",".join(line) + "\n")

                info_file.flush()

                info_file.write("\nDemand History:\n")
                i_str = [str(x) for x in range(i)]
                info_file.write("outer_iteration, (movers)," +
                                ",".join(i_str) + "\n")
                #info_file.write(", ,\n")
                for row in range(J):
                    line = [str(x) for x in demand_history[row, ]]
                    info_file.write(
                        str(alt_id[row]) + "," + ",".join(line) + "\n")

                demand_history_info_criteria = [500, 100, 50, 20]
                for criterion in demand_history_info_criteria:
                    com_rows_index = where(movers <= criterion)[0]
                    info_file.write(
                        "\nDemand History for alternatives with less than or equal to %s movers in 1998:\n"
                        % criterion)
                    i_str = [str(x) for x in range(i)]
                    info_file.write("outer_iteration, (movers)," +
                                    ",".join(i_str) + "\n")
                    #info_file.write(", movers,\n")
                    for row in com_rows_index:
                        line = [str(x) for x in demand_history[row, ]]
                        info_file.write(
                            str(alt_id[row]) + "," + ",".join(line) + "\n")

                #import pdb; pdb.set_trace()
                #export prob correlation history
                correlation_indices, prob_correlation = self.compute_prob_correlation(
                    probability_old, probability, prob_hat, index, resources)

                info_file.write("\nCorrelation of Probabilities:\n")
                c_name = [
                    'corr(p_ij p~_ij)', 'corr(p_ij p^_ij)', 'corr(p_ij dummy)',
                    'corr(p~_ij p^_ij)', 'corr(p~_ij dummy)',
                    'corr(p^_ij dummy)'
                ]

                info_file.write("com_id, " + ",".join(c_name) + "\n")

                #info_file.write(", ,\n")
                for row in range(correlation_indices.size):
                    line = [str(x) for x in prob_correlation[row, ]]
                    info_file.write(
                        str(alt_id[correlation_indices[row]]) + "," +
                        ",".join(line) + "\n")

                info_file.close()

                result['pi'] = pi
                return result

        logger.end_block()
        try:
            info_file.close()
        except:
            pass

        raise RuntimeError, "max iteration reached without convergence."
Exemple #36
0
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     values_sqft_per_job = ds.get_attribute(self.sqft_per_job)
     values_sqft = ds.get_attribute(self.sqft)
     return safe_array_divide(values_sqft, values_sqft_per_job, type="int32")
Exemple #37
0
 def compute(self, dataset_pool):
     return safe_array_divide(
         self.get_dataset().get_attribute(self.building_units),
         self.get_dataset().get_attribute(self.number_of_units))
 def compute(self, dataset_pool):
     return 100 * safe_array_divide(
         self.get_dataset().get_attribute(self.low_income),
         self.get_dataset().get_attribute(self.number_of_households),
     )
Exemple #39
0
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     return safe_array_divide(ds.get_attribute(self.jobs), 
                              ds.get_attribute(self.acres_of_land))
 def compute(self, dataset_pool):
     ds = self.get_dataset()
     return safe_array_divide(ds["number_of_jobs_of_sector_%s" % self.sector_id], ds["number_of_jobs"])
Exemple #41
0
 def compute(self, dataset_pool):
     acres = self.get_dataset().get_attribute(self.acres)
     return safe_array_divide(self.get_dataset().get_attribute(self.land_value),acres)
Exemple #42
0
 def compute(self, dataset_pool):
     nou = self.get_dataset().get_attribute("buildings_%s_space" %
                                            self.type)
     return safe_array_divide(
         self.get_dataset().get_attribute("total_value_%s" % self.type),
         nou)
 def compute(self, dataset_pool):
     vl = self.get_dataset().get_attribute("vacant_land_sqft").astype("float32")
     return safe_array_divide(
         self.get_dataset().sum_dataset_over_ids(dataset_pool.get_dataset("gridcell"), "total_value_vacant_land"), vl
     )
    def run(self, data, upc_sequence, resources=None):

        self.mnl_probabilities=upc_sequence.probability_class
        self.bhhh_estimation = bhhh_mnl_estimation()

        modified_upc_sequence = UPCFactory().get_model(
            utilities=None, probabilities="opus_core.mnl_probabilities", choices=None)
        modified_upc_sequence.utility_class = upc_sequence.utility_class

        N, neqs, V = data.shape

        max_iter = resources.get("max_iterations", 100)  # default
        sc = SessionConfiguration()
        dataset_pool = sc.get_dataset_pool()
        sample_rate = dataset_pool.get_dataset("sample_rate")
        
        CLOSE = sc["CLOSE"]
        info_filename = sc["info_file"]
        info_filename = os.path.join('.', info_filename)
        info_file = open(info_filename, "a")
        constraint_dict = {1:'constrained', 0:'unconstrained'}
        swing_cases_fix = 0  #set swing alternatives to constrained (1) or unconstrained (0)
        prob_correlation = None
        
        choice_set = resources['_model_'].choice_set
        J = choice_set.size()
        alt_id = choice_set.get_id_attribute()
        movers = choice_set.get_attribute('movers')

        resources.check_obligatory_keys(["capacity_string"])
        supply = choice_set.get_attribute(resources["capacity_string"])

        index = resources.get("index", None)
        if index is None: # no sampling case, alternative set is the full choice_set
            index = arange(J)
        if index.ndim <= 1:
            index = repeat(index[newaxis,:], N, axis=0)

        if resources.get('aggregate_to_dataset', None):
            aggregate_dataset = dataset_pool.get_dataset(resources.get('aggregate_to_dataset'))
            choice_set_aggregate_id = choice_set.get_attribute(aggregate_dataset.get_id_name()[0])
            index = aggregate_dataset.get_id_index(choice_set_aggregate_id[index].ravel()).reshape(index.shape)

            supply = aggregate_dataset.get_attribute(resources["capacity_string"])
            J = aggregate_dataset.size()

            movers = aggregate_dataset.get_attribute("movers")

        demand_history = movers[:, newaxis]
        resources.merge({"index":index})
        
        pi = ones(index.shape, dtype=float32)  #initialize pi
        #average_omega = ones(J,dtype=float32)  #initialize average_omega
        logger.start_block('Outer Loop')
        for i in range(max_iter):
            logger.log_status('Outer Loop Iteration %s' % i)

            result = self.bhhh_estimation.run(data, modified_upc_sequence, resources)
            del self.bhhh_estimation; collect()
            self.bhhh_estimation = bhhh_mnl_estimation()

            probability = modified_upc_sequence.get_probabilities()
            if data.shape[2] == V:  #insert a placeholder for ln(pi) in data
                data = concatenate((data,ones((N,neqs,1),dtype=float32)), axis=2)
                coef_names = resources.get("coefficient_names")
                coef_names = concatenate( (coef_names, array(["ln_pi"])) )
                resources.merge({"coefficient_names":coef_names})
            else:
                beta_ln_pi = result['estimators'][where(coef_names == 'ln_pi')][0]
                logger.log_status("mu = 1/%s = %s" % (beta_ln_pi, 1/beta_ln_pi))
                
                prob_hat = safe_array_divide(probability, pi ** beta_ln_pi)
                #prob_hat = safe_array_divide(probability, pi)
                prob_hat_sum = prob_hat.sum(axis=1, dtype=float32)
                if not ma.allclose(prob_hat_sum, 1.0):
                    logger.log_status("probability doesn't sum up to 1, with minimum %s, and maximum %s" %
                                      (prob_hat_sum.min(), prob_hat_sum.max()))
                    
                    probability = normalize(prob_hat)

            demand = self.mnl_probabilities.get_demand(index, probability, J) * 1 / sample_rate
            demand_history = concatenate((demand_history,
                                          demand[:, newaxis]),
                                          axis=1)

            sdratio = safe_array_divide(supply, demand, return_value_if_denominator_is_zero=2.0)
            sdratio_matrix = sdratio[index]
            ## debug info
            from numpy import histogram 
            from opus_core.misc import unique
            cc = histogram(index.ravel(), unique(index.ravel()))[0]
            logger.log_status( "=================================================================")
            logger.log_status( "Probability min: %s, max: %s" % (probability.min(), probability.max()) )
            logger.log_status( "Demand min: %s, max: %s" % (demand.min(), demand.max()) )
            logger.log_status( "sdratio min: %s, max: %s" % (sdratio.min(), sdratio.max()) )
            logger.log_status( "demand[sdratio==sdratio.min()]=%s" % demand[sdratio==sdratio.min()] )
            logger.log_status( "demand[sdratio==sdratio.max()]=%s" % demand[sdratio==sdratio.max()] )
            logger.log_status( "Counts of unique submarkets in alternatives min: %s, max: %s" % (cc.min(), cc.max()) )
            logger.log_status( "=================================================================")

            constrained_locations_matrix, omega, info = self.inner_loop(supply, demand, probability,
                                                                        index, sdratio_matrix,
                                                                        J, max_iteration=max_iter)

            inner_iterations, constrained_locations_history, swing_index, average_omega_history = info
    
            for idx in swing_index:
                logger.log_status("swinging alt with id %s set to %s" % (alt_id[idx], constraint_dict[swing_cases_fix]))
                constrained_locations_matrix[index==idx] = swing_cases_fix
    
            if swing_index.size > 0:    
                info_file.write("swing of constraints found with id %s \n" % alt_id[swing_index])
                info_file.write("outer_iteration, %i, " % i + ", ".join([str(i)]*(len(inner_iterations))) + "\n")
                info_file.write("inner_iteration, , " + ", ".join(inner_iterations) + "\n")
                info_file.write("id, sdratio, " + ", ".join(["avg_omega"]*len(inner_iterations)) + "\n")
                for idx in swing_index:
                    line = str(alt_id[idx]) + ','
                    line += str(sdratio[idx]) + ','
                    line += ",".join([str(x) for x in average_omega_history[idx,]])
                    line += "\n"
                    info_file.write(line)
    
                info_file.write("\n")
                info_file.flush()

            outer_iterations = [str(i)] * len(inner_iterations)
            prob_min = [str(probability.min())] * len(inner_iterations)
            prob_max = [str(probability.max())] * len(inner_iterations)

            pi_new = self.mnl_probabilities.get_pi(sdratio_matrix, omega, constrained_locations_matrix)

            data[:,:,-1] = ln(pi_new)
            #diagnostic output
            
            if not ma.allclose(pi, pi_new, atol=CLOSE):
                if i > 0:  #don't print this for the first iteration
                    logger.log_status("min of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).min())
                    logger.log_status("max of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).max())
                    logger.log_status("mean of pi(l+1) - pi(l): %s" % (pi_new - pi).mean())
                    logger.log_status('Standard Deviation pi(l+1) - pi(l): %s' % standard_deviation(pi_new - pi))
                    logger.log_status('correlation of pi(l+1) and pi(l): %s' % corr(pi_new.ravel(), pi.ravel())[0,1])

                pi = pi_new
                probability_old = probability   # keep probability of the previous loop, for statistics computation only    
            else:   #convergence criterion achieved, quiting outer loop
                logger.log_status("pi(l) == pi(l+1): Convergence criterion achieved")
    
                info_file.write("\nConstrained Locations History:\n")
                info_file.write("outer_iteration," + ",".join(outer_iterations) + "\n")
                info_file.write("inner_iteration," + ",".join(inner_iterations) + "\n")
                info_file.write("minimum_probability," + ",".join(prob_min) + "\n")
                info_file.write("maximum_probability," + ",".join(prob_max) + "\n")
                for row in range(J):
                    line = [str(x) for x in constrained_locations_history[row,]]
                    info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n")

                info_file.flush()

                info_file.write("\nDemand History:\n")
                i_str = [str(x) for x in range(i)]
                info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n")
                #info_file.write(", ,\n")
                for row in range(J):
                    line = [str(x) for x in demand_history[row,]]
                    info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n")

                demand_history_info_criteria = [500, 100, 50, 20]
                for criterion in demand_history_info_criteria:
                    com_rows_index = where(movers <= criterion)[0]
                    info_file.write("\nDemand History for alternatives with less than or equal to %s movers in 1998:\n" % criterion)
                    i_str = [str(x) for x in range(i)]
                    info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n")
                    #info_file.write(", movers,\n")
                    for row in com_rows_index:
                        line = [str(x) for x in demand_history[row,]]
                        info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n")

                #import pdb; pdb.set_trace()
                #export prob correlation history
                correlation_indices, prob_correlation = self.compute_prob_correlation(probability_old, probability, prob_hat, index, resources)

                info_file.write("\nCorrelation of Probabilities:\n")
                c_name = ['corr(p_ij p~_ij)', 'corr(p_ij p^_ij)', 'corr(p_ij dummy)', 'corr(p~_ij p^_ij)', 'corr(p~_ij dummy)', 'corr(p^_ij dummy)']

                info_file.write("com_id, " + ",".join(c_name) + "\n")

                #info_file.write(", ,\n")
                for row in range(correlation_indices.size):
                    line = [str(x) for x in prob_correlation[row,]]
                    info_file.write(str(alt_id[correlation_indices[row]]) + "," + ",".join(line) + "\n")

                info_file.close()

                result['pi'] = pi
                return result

        logger.end_block()
        try:info_file.close()
        except:pass

        raise RuntimeError, "max iteration reached without convergence."
    def _add(self, agents_pool, amount, agent_dataset, location_dataset,
             this_refinement, dataset_pool):

        fit_index = self.get_fit_agents_index(
            agent_dataset, this_refinement.agent_expression,
            this_refinement.location_expression, dataset_pool)
        movers_index = array([], dtype="int32")
        amount_from_agents_pool = min(amount, len(agents_pool))
        if amount_from_agents_pool > 0:
            agents_index_from_agents_pool = sample_noreplace(
                agents_pool, amount_from_agents_pool)
            [agents_pool.remove(i) for i in agents_index_from_agents_pool]
            if fit_index.size == 0:
                ##cannot find agents to copy their location or clone them, place agents in agents_pool
                logger.log_warning("Refinement requests to add %i agents,  but there are only %i agents subtracted from previous action(s) and no agents satisfying %s to clone from;" \
                                   "add %i agents instead" % (amount, amount_from_agents_pool,
                                                              ' and '.join( [this_refinement.agent_expression,
                                                                           this_refinement.location_expression]).strip(' and '),
                                                              amount_from_agents_pool,) )

                amount = amount_from_agents_pool

                is_suitable_location = location_dataset.compute_variables(
                    this_refinement.location_expression,
                    dataset_pool=dataset_pool)
                location_id_for_agents_pool = sample_replace(
                    location_dataset.get_id_attribute()[is_suitable_location],
                    amount_from_agents_pool)
            else:

                agents_index_for_location = sample_replace(
                    fit_index, amount_from_agents_pool)
                location_id_for_agents_pool = agent_dataset.get_attribute(
                    location_dataset.get_id_name()
                    [0])[agents_index_for_location]
                movers_index = concatenate(
                    (movers_index, agents_index_for_location))

        elif fit_index.size == 0:
            ## no agents in agents_pool and no agents to clone either, --> fail
            logger.log_error( "Action 'add' failed: there is no agent subtracted from previous action, and no suitable agents satisfying %s to clone from." % \
                              ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip('and') )
            return

        if amount > amount_from_agents_pool:
            agents_index_to_clone = sample_replace(
                fit_index, amount - amount_from_agents_pool)
            movers_index = concatenate((movers_index, agents_index_to_clone))

        if movers_index.size > 0 and this_refinement.location_capacity_attribute is not None and len(
                this_refinement.location_capacity_attribute) > 0:
            movers_location_id = agent_dataset.get_attribute(
                location_dataset.get_id_name()[0])[movers_index]
            movers_location_index = location_dataset.get_id_index(
                movers_location_id)
            # see previous comment about histogram function
            num_of_movers_by_location = histogram(
                movers_location_index,
                bins=arange(location_dataset.size() + 1))[0]
            num_of_agents_by_location = location_dataset.compute_variables( "number_of_agents=%s.number_of_agents(%s)" % \
                                                                            ( location_dataset.dataset_name,
                                                                            agent_dataset.dataset_name ),
                                                                            dataset_pool=dataset_pool)

            expand_factor = safe_array_divide(
                (num_of_agents_by_location +
                 num_of_movers_by_location).astype('float32'),
                num_of_agents_by_location,
                return_value_if_denominator_is_zero=1.0)
            new_values = round_(expand_factor * location_dataset.get_attribute(
                this_refinement.location_capacity_attribute))
            location_dataset.modify_attribute(
                this_refinement.location_capacity_attribute, new_values)
            self._add_refinement_info_to_dataset(location_dataset,
                                                 self.id_names,
                                                 this_refinement,
                                                 index=movers_location_index)
        if amount_from_agents_pool > 0:
            agent_dataset.modify_attribute(location_dataset.get_id_name()[0],
                                           location_id_for_agents_pool,
                                           agents_index_from_agents_pool)
            self._add_refinement_info_to_dataset(
                agent_dataset,
                self.id_names,
                this_refinement,
                index=agents_index_from_agents_pool)
        if amount > amount_from_agents_pool:
            new_agents_index = agent_dataset.duplicate_rows(
                agents_index_to_clone)
            self._add_refinement_info_to_dataset(agent_dataset,
                                                 self.id_names,
                                                 this_refinement,
                                                 index=agents_index_to_clone)
            self._add_refinement_info_to_dataset(agent_dataset,
                                                 self.id_names,
                                                 this_refinement,
                                                 index=new_agents_index)