def get_households_for_estimation(agent_set, in_storage, agents_for_estimation_table_name, exclude_condition=None, join_datasets=True): estimation_set = Dataset(in_storage=in_storage, in_table_name=agents_for_estimation_table_name, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) agent_set.unload_primary_attributes() agent_set.load_dataset(attributes='*') estimation_set.load_dataset( attributes=agent_set.get_primary_attribute_names()) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size()) else: index = agent_set.get_id_index(estimation_set.get_id_attribute()) exclude_ids = [] if exclude_condition is not None: exclude_ids = agent_set.get_id_attribute()[where( agent_set.compute_variables(exclude_condition))] for id in exclude_ids: minus = agent_set.get_id_index(id) if minus in index: index = index[index != minus] return (agent_set, index)
def get_households_for_estimation(agent_set, in_storage, agents_for_estimation_table_name, exclude_condition=None, join_datasets=True): estimation_set = Dataset(in_storage = in_storage, in_table_name=agents_for_estimation_table_name, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) agent_set.unload_primary_attributes() agent_set.load_dataset(attributes='*') estimation_set.load_dataset(attributes=agent_set.get_primary_attribute_names()) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size()-estimation_set.size(),agent_set.size()) else: index = agent_set.get_id_index(estimation_set.get_id_attribute()) exclude_ids = [] if exclude_condition is not None: exclude_ids = agent_set.get_id_attribute()[where(agent_set.compute_variables(exclude_condition))] for id in exclude_ids: minus = agent_set.get_id_index(id) if minus in index: index = index[index != minus] return (agent_set, index)
def prepare_for_estimate(self, specification_dict=None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, index_to_unplace=None, portion_to_unplace=1.0, agent_filter=None, data_objects={}): from opus_core.model import get_specification_for_estimation specification = get_specification_for_estimation( specification_dict, specification_storage, specification_table) if (agent_set is not None) and (index_to_unplace is not None): if self.location_id_string is not None: agent_set.compute_variables(self.location_id_string, resources=Resources(data_objects)) if portion_to_unplace < 1: unplace_size = int(portion_to_unplace * index_to_unplace.size) end_index_to_unplace = sample_noreplace( index_to_unplace, unplace_size) else: end_index_to_unplace = index_to_unplace logger.log_status("Unplace " + str(end_index_to_unplace.size) + " agents.") agent_set.modify_attribute(self.choice_set.get_id_name()[0], -1 * ones(end_index_to_unplace.size), end_index_to_unplace) # create agents for estimation if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage=agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) if agent_filter is not None: estimation_set.compute_variables( agent_filter, resources=Resources(data_objects)) index = where( estimation_set.get_attribute(agent_filter) > 0)[0] estimation_set.subset_by_index( index, flush_attributes_if_not_loaded=False) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size()) else: index = agent_set.get_id_index( estimation_set.get_id_attribute()) else: index = arange(agent_set.size()) return (specification, index)
def prepare_for_estimate(specification_dict=None, specification_storage=None, specification_table=None, agent_set=None, household_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, households_for_estimation_table=None, join_datasets=False, filter=None, data_objects=None): specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage=agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) hh_estimation_set = None if households_for_estimation_table is not None: hh_estimation_set = Dataset( in_storage=agents_for_estimation_storage, in_table_name=households_for_estimation_table, id_name=household_set.get_id_name(), dataset_name=household_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if filter: estimation_set.compute_variables(filter, resources=Resources(data_objects)) filter_index = where(estimation_set.get_attribute(filter) > 0)[0] #estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: if hh_estimation_set is not None: household_set.join_by_rows(hh_estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index] else: index = agent_set.get_id_index( estimation_set.get_id_attribute()[filter_index]) else: if agent_set is not None: index = arange(agent_set.size()) else: index = None return (specification, index)
def prepare_for_estimate(specification_dict = None, specification_storage=None, specification_table=None, agent_set=None, household_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, households_for_estimation_table=None, join_datasets=False, filter=None, data_objects=None): specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) hh_estimation_set = None if households_for_estimation_table is not None: hh_estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=households_for_estimation_table, id_name=household_set.get_id_name(), dataset_name=household_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if filter: estimation_set.compute_variables(filter, resources=Resources(data_objects)) filter_index = where(estimation_set.get_attribute(filter) > 0)[0] #estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: if hh_estimation_set is not None: household_set.join_by_rows(hh_estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index] else: index = agent_set.get_id_index(estimation_set.get_id_attribute()[filter_index]) else: if agent_set is not None: index = arange(agent_set.size()) else: index = None return (specification, index)
def prepare_for_estimate(specification_dict=None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, filter=None, agents_filter=None, data_objects=None): """ filter - alias to agents_filter for backforward compatibility, which is more specific """ if agents_filter is None and filter is not None: agents_filter = filter specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage=agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if agents_filter: filter_condition = estimation_set.compute_variables(agents_filter, resources=Resources(data_objects)) filter_index = where(filter_condition)[0] if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index] else: index = agent_set.get_id_index(estimation_set.get_id_attribute()[filter_index]) else: if agent_set is not None: index = arange(agent_set.size()) else: index = None return (specification, index)
def prepare_for_estimate(specification_dict=None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, filter=None, agents_filter=None, data_objects=None): """ filter - alias to agents_filter for backforward compatibility, which is more specific """ if agents_filter is None and filter is not None: agents_filter = filter specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage=agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if agents_filter: filter_condition = estimation_set.compute_variables( agents_filter, resources=Resources(data_objects)) filter_index = where(filter_condition)[0] if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size())[filter_index] else: index = agent_set.get_id_index( estimation_set.get_id_attribute()[filter_index]) else: if agent_set is not None: index = arange(agent_set.size()) else: index = None return (specification, index)
def prepare_for_estimate(self, dataset=None, dataset_for_estimation_storage=None, dataset_for_estimation_table=None, join_datasets=False, **kwargs): from opus_core.model import get_specification_for_estimation from opus_core.datasets.dataset import Dataset spec = get_specification_for_estimation(**kwargs) if (dataset_for_estimation_storage is not None) and (dataset_for_estimation_table is not None): estimation_set = Dataset(in_storage = dataset_for_estimation_storage, in_table_name=dataset_for_estimation_table, id_name=dataset.get_id_name(), dataset_name=dataset.get_dataset_name()) if join_datasets: dataset.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(dataset.size()-estimation_set.size(),dataset.size()) else: index = dataset.get_id_index(estimation_set.get_id_attribute()) else: index = None return (spec, index)
def prepare_for_estimate(self, specification_dict = None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, index_to_unplace=None, portion_to_unplace=1.0, agent_filter=None, data_objects={}): from opus_core.models.model import get_specification_for_estimation specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if (agent_set is not None) and (index_to_unplace is not None): if self.location_id_string is not None: agent_set.compute_variables(self.location_id_string, resources=Resources(data_objects)) if portion_to_unplace < 1: unplace_size = int(portion_to_unplace*index_to_unplace.size) end_index_to_unplace = sample_noreplace(index_to_unplace, unplace_size) else: end_index_to_unplace = index_to_unplace logger.log_status("Unplace " + str(end_index_to_unplace.size) + " agents.") agent_set.modify_attribute(self.choice_set.get_id_name()[0], -1*ones(end_index_to_unplace.size), end_index_to_unplace) # create agents for estimation if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) if agent_filter is not None: estimation_set.compute_variables(agent_filter, resources=Resources(data_objects)) index = where(estimation_set.get_attribute(agent_filter) > 0)[0] estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size()-estimation_set.size(),agent_set.size()) else: index = agent_set.get_id_index(estimation_set.get_id_attribute()) else: index = arange(agent_set.size()) return (specification, index)
# Working with datasets import os import urbansim us_path = urbansim.__path__[0] from opus_core.storage_factory import StorageFactory storage = StorageFactory().get_storage('tab_storage', storage_location = os.path.join(us_path, "data/tutorial")) from opus_core.datasets.dataset import Dataset households = Dataset(in_storage = storage, in_table_name = 'households', id_name='household_id', dataset_name='household') households.get_attribute_names() households.get_id_attribute() households.size() households.get_attribute("income") households.get_attribute_names() households.load_dataset() households.get_attribute_names() #households.plot_histogram("income", bins = 10) #households.r_histogram("income") #households.r_scatter("persons", "income") households.correlation_coefficient("persons", "income") households.correlation_matrix(["persons", "income"]) households.summary() households.add_primary_attribute(data=[4,6,9,2,4,8,2,1,3,2], name="location") households.get_attribute_names() households.modify_attribute(name="location", data=[0,0], index=[0,1]) households.get_attribute("location")
def prepare_for_estimate(self, add_member_prefix=True, specification_dict=None, specification_storage=None, specification_table=None, building_set=None, buildings_for_estimation_storage=None, buildings_for_estimation_table=None, constants=None, base_year=0, building_categories=None, location_id_variable=None, join_datasets=False, data_objects=None, **kwargs): # buildings = None if (building_set is not None): if location_id_variable is not None: building_set.compute_variables( location_id_variable, resources=Resources(data_objects)) # create agents for estimation if buildings_for_estimation_storage is not None: estimation_set = Dataset( in_storage=buildings_for_estimation_storage, in_table_name=buildings_for_estimation_table, id_name=building_set.get_id_name(), dataset_name=building_set.get_dataset_name()) if location_id_variable: estimation_set.compute_variables( location_id_variable, resources=Resources(data_objects)) # needs to be a primary attribute because of the join method below estimation_set.add_primary_attribute( estimation_set.get_attribute(location_id_variable), VariableName(location_id_variable).alias()) years = estimation_set.get_attribute("scheduled_year") recent_years = constants['recent_years'] indicator = zeros(estimation_set.size()) for year in range(base_year - recent_years, base_year + 1): indicator = logical_or(indicator, years == year) idx = where(logical_not(indicator))[0] estimation_set.remove_elements(idx) #if filter: #estimation_set.compute_variables(filter, resources=Resources(data_objects)) #index = where(estimation_set.get_attribute(filter) > 0)[0] #estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: building_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(building_set.size() - estimation_set.size(), agent_set.size()) else: index = building_set.get_id_index( estimation_set.get_id_attribute()) else: if building_set is not None: index = arange(building_set.size()) else: index = None if add_member_prefix: specification_table = self.group_member.add_member_prefix_to_table_names( [specification_table]) from opus_core.model import get_specification_for_estimation #from urbansim.functions import compute_supply_and_add_to_location_set specification = get_specification_for_estimation( specification_dict, specification_storage, specification_table) #specification, dummy = AgentLocationChoiceModelMember.prepare_for_estimate(self, add_member_prefix, #specification_dict, specification_storage, #specification_table, #location_id_variable=location_id_variable, #data_objects=data_objects, **kwargs) return (specification, index)
def prepare_for_estimate(self, specification_dict = None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, index_to_unplace=None, portion_to_unplace=1.0, compute_lambda=False, grouping_location_set=None, movers_variable=None, movers_index=None, filter=None, location_id_variable=None, data_objects={}): """Put 'location_id_variable' always in, if the location id is to be computed on the estimation set, i.e. if it is not a primary attribute of the estimation set. Set 'index_to_unplace' to None, if 'compute_lambda' is True. In such a case, the annual supply is estimated without unplacing agents. 'grouping_location_set', 'movers_variable' and 'movers_index' must be given, if 'compute_lambda' is True. """ from opus_core.model import get_specification_for_estimation from urbansim.functions import compute_supply_and_add_to_location_set specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if (agent_set is not None) and (index_to_unplace is not None): if self.location_id_string is not None: agent_set.compute_variables(self.location_id_string, resources=Resources(data_objects)) if portion_to_unplace < 1: unplace_size = int(portion_to_unplace*index_to_unplace.size) end_index_to_unplace = sample_noreplace(index_to_unplace, unplace_size) else: end_index_to_unplace = index_to_unplace logger.log_status("Unplace " + str(end_index_to_unplace.size) + " agents.") agent_set.modify_attribute(self.choice_set.get_id_name()[0], resize(array([-1]), end_index_to_unplace.size), end_index_to_unplace) if compute_lambda: movers = zeros(agent_set.size(), dtype="bool8") if movers_index is not None: movers[movers_index] = 1 agent_set.add_primary_attribute(movers, "potential_movers") self.estimate_config["weights_for_estimation_string"] = self.estimate_config["weights_for_estimation_string"]+"_from_lambda" compute_supply_and_add_to_location_set(self.choice_set, grouping_location_set, self.run_config["number_of_units_string"], self.run_config["capacity_string"], movers_variable, self.estimate_config["weights_for_estimation_string"], resources=Resources(data_objects)) # create agents for estimation if (agents_for_estimation_storage is not None) and (agents_for_estimation_table is not None): estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) if location_id_variable is not None: estimation_set.compute_variables(location_id_variable, resources=Resources(data_objects)) # needs to be a primary attribute because of the join method below estimation_set.add_primary_attribute(estimation_set.get_attribute(location_id_variable), VariableName(location_id_variable).get_alias()) if filter: values = estimation_set.compute_variables(filter, resources=Resources(data_objects)) index = where(values > 0)[0] estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size()-estimation_set.size(),agent_set.size()) else: index = agent_set.get_id_index(estimation_set.get_id_attribute()) else: if agent_set is not None: if filter is not None: values = agent_set.compute_variables(filter, resources=Resources(data_objects)) index = where(values > 0)[0] else: index = arange(agent_set.size()) else: index = None return (specification, index)
def prepare_for_estimate( self, add_member_prefix=True, specification_dict=None, specification_storage=None, specification_table=None, building_set=None, buildings_for_estimation_storage=None, buildings_for_estimation_table=None, constants=None, base_year=0, building_categories=None, location_id_variable=None, join_datasets=False, data_objects=None, **kwargs ): # buildings = None if building_set is not None: if location_id_variable is not None: building_set.compute_variables(location_id_variable, resources=Resources(data_objects)) # create agents for estimation if buildings_for_estimation_storage is not None: estimation_set = Dataset( in_storage=buildings_for_estimation_storage, in_table_name=buildings_for_estimation_table, id_name=building_set.get_id_name(), dataset_name=building_set.get_dataset_name(), ) if location_id_variable: estimation_set.compute_variables(location_id_variable, resources=Resources(data_objects)) # needs to be a primary attribute because of the join method below estimation_set.add_primary_attribute( estimation_set.get_attribute(location_id_variable), VariableName(location_id_variable).alias() ) years = estimation_set.get_attribute("scheduled_year") recent_years = constants["recent_years"] indicator = zeros(estimation_set.size(), dtype="int32") for year in range(base_year - recent_years, base_year + 1): indicator = logical_or(indicator, years == year) idx = where(logical_not(indicator))[0] estimation_set.remove_elements(idx) # if filter: # estimation_set.compute_variables(filter, resources=Resources(data_objects)) # index = where(estimation_set.get_attribute(filter) > 0)[0] # estimation_set.subset_by_index(index, flush_attributes_if_not_loaded=False) if join_datasets: building_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(building_set.size() - estimation_set.size(), building_set.size()) else: index = building_set.get_id_index(estimation_set.get_id_attribute()) else: if building_set is not None: index = arange(building_set.size()) else: index = None if add_member_prefix: specification_table = self.group_member.add_member_prefix_to_table_names([specification_table]) from opus_core.model import get_specification_for_estimation # from urbansim.functions import compute_supply_and_add_to_location_set specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) # specification, dummy = AgentLocationChoiceModelMember.prepare_for_estimate(self, add_member_prefix, # specification_dict, specification_storage, # specification_table, # location_id_variable=location_id_variable, # data_objects=data_objects, **kwargs) return (specification, index)
def prepare_for_estimate(self, specification_dict=None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, join_datasets=False, index_to_unplace=None, portion_to_unplace=1.0, compute_lambda=False, grouping_location_set=None, movers_variable=None, movers_index=None, filter=None, location_id_variable=None, data_objects={}): """Put 'location_id_variable' always in, if the location id is to be computed on the estimation set, i.e. if it is not a primary attribute of the estimation set. Set 'index_to_unplace' to None, if 'compute_lambda' is True. In such a case, the annual supply is estimated without unplacing agents. 'grouping_location_set', 'movers_variable' and 'movers_index' must be given, if 'compute_lambda' is True. """ from opus_core.model import get_specification_for_estimation from urbansim.functions import compute_supply_and_add_to_location_set specification = get_specification_for_estimation( specification_dict, specification_storage, specification_table) if (agent_set is not None) and (index_to_unplace is not None): if self.location_id_string is not None: agent_set.compute_variables(self.location_id_string, resources=Resources(data_objects)) if portion_to_unplace < 1: unplace_size = int(portion_to_unplace * index_to_unplace.size) end_index_to_unplace = sample_noreplace( index_to_unplace, unplace_size) else: end_index_to_unplace = index_to_unplace logger.log_status("Unplace " + str(end_index_to_unplace.size) + " agents.") agent_set.modify_attribute( self.choice_set.get_id_name()[0], resize(array([-1]), end_index_to_unplace.size), end_index_to_unplace) if compute_lambda: movers = zeros(agent_set.size(), dtype="bool8") if movers_index is not None: movers[movers_index] = 1 agent_set.add_primary_attribute(movers, "potential_movers") self.estimate_config[ "weights_for_estimation_string"] = self.estimate_config[ "weights_for_estimation_string"] + "_from_lambda" compute_supply_and_add_to_location_set( self.choice_set, grouping_location_set, self.run_config["number_of_units_string"], self.run_config["capacity_string"], movers_variable, self.estimate_config["weights_for_estimation_string"], resources=Resources(data_objects)) # create agents for estimation if (agents_for_estimation_storage is not None) and (agents_for_estimation_table is not None): estimation_set = Dataset(in_storage=agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name()) if location_id_variable is not None: estimation_set.compute_variables( location_id_variable, resources=Resources(data_objects)) # needs to be a primary attribute because of the join method below estimation_set.add_primary_attribute( estimation_set.get_attribute(location_id_variable), VariableName(location_id_variable).get_alias()) if filter: values = estimation_set.compute_variables( filter, resources=Resources(data_objects)) index = where(values > 0)[0] estimation_set.subset_by_index( index, flush_attributes_if_not_loaded=False) if join_datasets: agent_set.join_by_rows(estimation_set, require_all_attributes=False, change_ids_if_not_unique=True) index = arange(agent_set.size() - estimation_set.size(), agent_set.size()) else: index = agent_set.get_id_index( estimation_set.get_id_attribute()) else: if agent_set is not None: if filter is not None: values = agent_set.compute_variables( filter, resources=Resources(data_objects)) index = where(values > 0)[0] else: index = arange(agent_set.size()) else: index = None return (specification, index)