def test_agent_times_choice(self):
expression = 'agent_x_choice.agent_times_choice(attr)'
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='agents',
table_data={'id': array([1, 2, 3, 4, 5]), 'attr_2': array([3, 2, 4, 10, 20]),
'attr_3': array([10, 100, 1000, 500, 0]),
'attr_4': array([100, 500, 0, 20, -30])
}
)
storage.write_table(table_name='choices',
table_data={'id': array([1, 2, 3, 4])}
)
agents = Dataset(in_storage=storage, in_table_name='agents', dataset_name='agent', id_name='id')
choices = Dataset(in_storage=storage, in_table_name='choices', dataset_name='choice', id_name='id')
ids = InteractionDataset(dataset1=agents, dataset2=choices, index1=array([0,1,3,4]), index2=array([1,2,3]))
result = ids.compute_variables(expression)
should_be = array([[3, 10, 100], [2,100,500], [10,500, 20], [20, 0, -30]])
self.assertEqual(ma.allequal(result, should_be), True)
agents.touch_attribute('attr_2') # in order to recompute the expression
choices.add_primary_attribute(name='name', data=array(['bus', 'car', 'tran', 'walk']))
agents.add_primary_attribute(name='attr_tran', data=array([100, 1000, 10000, 5000,10]))
result = ids.compute_variables(expression)
should_be = array([[3, 100, 100], [2,1000,500], [10,5000, 20], [20, 10, -30]])
self.assertEqual(ma.allequal(result, should_be), True)
def test_agent_times_choice(self):
expression = "agent_x_choice.agent_times_choice(attr)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="agents",
table_data={
"id": array([1, 2, 3, 4, 5]),
"attr_2": array([3, 2, 4, 10, 20]),
"attr_3": array([10, 100, 1000, 500, 0]),
"attr_4": array([100, 500, 0, 20, -30]),
},
)
storage.write_table(table_name="choices", table_data={"id": array([1, 2, 3, 4])})
agents = Dataset(in_storage=storage, in_table_name="agents", dataset_name="agent", id_name="id")
choices = Dataset(in_storage=storage, in_table_name="choices", dataset_name="choice", id_name="id")
ids = InteractionDataset(dataset1=agents, dataset2=choices, index1=array([0, 1, 3, 4]), index2=array([1, 2, 3]))
result = ids.compute_variables(expression)
should_be = array([[3, 10, 100], [2, 100, 500], [10, 500, 20], [20, 0, -30]])
self.assertEqual(ma.allequal(result, should_be), True)
agents.touch_attribute("attr_2") # in order to recompute the expression
choices.add_primary_attribute(name="name", data=array(["bus", "car", "tran", "walk"]))
agents.add_primary_attribute(name="attr_tran", data=array([100, 1000, 10000, 5000, 10]))
result = ids.compute_variables(expression)
should_be = array([[3, 100, 100], [2, 1000, 500], [10, 5000, 20], [20, 10, -30]])
self.assertEqual(ma.allequal(result, should_be), True)
def __init__(self, resources=None, dataset1=None, dataset2=None, index1 = None, index2 = None,
debuglevel=0):
debug = DebugPrinter(debuglevel)
debug.print_debug("Creating object %s.%s" % (self.__class__.__module__, self.__class__.__name__), 2)
local_resources = Resources(resources)
local_resources.merge_if_not_None({"dataset1":dataset1,
"dataset2":dataset2, "debug":debug,
"index1":index1, "index2":index2})
CoreInteractionDataset.__init__(self, resources = local_resources)
def create_interaction_dataset(self,
dataset1,
dataset2,
index1=None,
index2=None):
return InteractionDataset(dataset1=dataset1,
dataset2=dataset2,
index1=index1,
index2=index2)
def __init__(self,
resources=None,
dataset1=None,
dataset2=None,
index1=None,
index2=None,
debuglevel=0):
debug = DebugPrinter(debuglevel)
debug.print_debug(
"Creating object %s.%s" %
(self.__class__.__module__, self.__class__.__name__), 2)
local_resources = Resources(resources)
local_resources.merge_if_not_None({
"dataset1": dataset1,
"dataset2": dataset2,
"debug": debug,
"index1": index1,
"index2": index2
})
CoreInteractionDataset.__init__(self, resources=local_resources)
data=[85,20,0,90,35,51,0,10,5])
from opus_core.datasets.dataset_pool import DatasetPool
dataset_pool = DatasetPool(package_order=['urbansim', 'opus_core'], storage=storage)
dataset_pool.datasets_in_pool()
hs = dataset_pool.get_dataset("household")
dataset_pool.datasets_in_pool()
hs.size()
constant = dataset_pool.get_dataset("urbansim_constant")
constant["percent_coverage_threshold"] = 50
locations.compute_variables(["urbansim.gridcell.is_in_wetland"], dataset_pool=dataset_pool)
locations.get_attribute("is_in_wetland")
# Interaction variables
from opus_core.datasets.interaction_dataset import InteractionDataset
interactions = InteractionDataset(dataset1 = households, dataset2 = locations)
interactions.get_dataset_name()
from numpy import arange
interactions = InteractionDataset(dataset1 = households, dataset2 = locations, index1 = arange(5), index2 = arange(3))
interactions.compute_variables(["urbansim.household_x_gridcell.cost_times_income"])
specification = EquationSpecification(
variables=array([
"gridcell.cost",
"urbansim.household_x_gridcell.cost_times_income"]),
coefficients=array(["costcoef", "cti_coef"]))
households.add_primary_attribute(data=[2,8,3,1,5,4,9,7,3,6], name="location")
coef, other_results = hlcm.estimate(specification, households)
# Versioning
def run(self,
dataset1,
dataset2,
index1=None,
index2=None,
sample_size=10,
weight=None,
include_chosen_choice=False,
with_replacement=False,
resources=None,
dataset_pool=None):
"""this function samples number of sample_size (scalar value) alternatives from dataset2
for agent set specified by dataset1.
If index1 is not None, only samples alterantives for agents with indices in index1;
if index2 is not None, only samples alternatives from indices in index2.
sample_size specifies number of alternatives to be sampled for each agent.
weight, to be used as sampling weight, is either an attribute name of dataset2, or a 1d
array of the same length as index2 or 2d array of shape (index1.size, index2.size).
Also refer to document of interaction_dataset"""
if dataset_pool is None:
try:
sc = SessionConfiguration()
dataset_pool = sc.get_dataset_pool()
except:
dataset_pool = DatasetPool()
local_resources = Resources(resources)
local_resources.merge_if_not_None({
"dataset1":
dataset1,
"dataset2":
dataset2,
"index1":
index1,
"index2":
index2,
"sample_size":
sample_size,
"weight":
weight,
"with_replacement":
with_replacement,
"include_chosen_choice":
include_chosen_choice
})
local_resources.check_obligatory_keys(
['dataset1', 'dataset2', 'sample_size'])
agent = local_resources["dataset1"]
index1 = local_resources.get("index1", None)
if index1 is None:
index1 = arange(agent.size())
choice = local_resources["dataset2"]
index2 = local_resources.get("index2", None)
if index2 is None:
index2 = arange(choice.size())
if index1.size == 0 or index2.size == 0:
err_msg = "either choice size or agent size is zero, return None"
logger.log_warning(err_msg)
return None
include_chosen_choice = local_resources.get("include_chosen_choice",
False)
J = local_resources["sample_size"]
if include_chosen_choice:
J = J - 1
with_replacement = local_resources.get("with_replacement")
weight = local_resources.get("weight", None)
if isinstance(weight, str):
if weight in choice.get_known_attribute_names():
weight = choice.get_attribute(weight)
rank_of_weight = 1
else:
varname = VariableName(weight)
if varname.get_dataset_name() == choice.get_dataset_name():
weight = choice.compute_variables(
weight, dataset_pool=dataset_pool)
rank_of_weight = 1
elif varname.get_interaction_set_names() is not None:
## weights can be an interaction variable
interaction_dataset = InteractionDataset(local_resources)
weight = interaction_dataset.compute_variables(
weight, dataset_pool=dataset_pool)
rank_of_weight = 2
assert (len(weight.shape) >= rank_of_weight)
else:
err_msg = ("weight is neither a known attribute name "
"nor a simple variable from the choice dataset "
"nor an interaction variable: '%s'" % weight)
logger.log_error(err_msg)
raise ValueError, err_msg
elif isinstance(weight, ndarray):
rank_of_weight = weight.ndim
elif not weight: ## weight is None or empty string
weight = ones(index2.size)
rank_of_weight = 1
else:
err_msg = "unkown weight type"
logger.log_error(err_msg)
raise TypeError, err_msg
if (weight.size <> index2.size) and (weight.shape[rank_of_weight - 1]
<> index2.size):
if weight.shape[rank_of_weight - 1] == choice.size():
if rank_of_weight == 1:
weight = take(weight, index2)
if rank_of_weight == 2:
weight = take(weight, index2, axis=1)
else:
err_msg = "weight array size doesn't match to size of dataset2 or its index"
logger.log_error(err_msg)
raise ValueError, err_msg
prob = normalize(weight)
#chosen_choice = ones(index1.size) * UNPLACED_ID
chosen_choice_id = agent.get_attribute(choice.get_id_name()[0])[index1]
#index_of_placed_agent = where(greater(chosen_choice_id, UNPLACED_ID))[0]
chosen_choice_index = choice.try_get_id_index(
chosen_choice_id, return_value_if_not_found=UNPLACED_ID)
chosen_choice_index_to_index2 = lookup(chosen_choice_index,
index2,
index_if_not_found=UNPLACED_ID)
if rank_of_weight == 1: # if weight_array is 1d, then each agent shares the same weight for choices
replace = with_replacement # sampling with no replacement
non_zero_counts = nonzerocounts(weight)
if non_zero_counts < J:
logger.log_warning(
"weight array dosen't have enough non-zero counts, use sample with replacement"
)
replace = True
if non_zero_counts > 0:
sampled_index = prob2dsample(
index2,
sample_size=(index1.size, J),
prob_array=prob,
exclude_index=chosen_choice_index_to_index2,
replace=replace,
return_index=True)
else:
# all alternatives have a zero weight
sampled_index = zeros((index1.size, 0), dtype=DTYPE)
#return index2[sampled_index]
if rank_of_weight == 2:
sampled_index = zeros((index1.size, J), dtype=DTYPE) - 1
for i in range(index1.size):
replace = with_replacement # sampling with/without replacement
i_prob = prob[i, :]
if nonzerocounts(i_prob) < J:
logger.log_warning(
"weight array dosen't have enough non-zero counts, use sample with replacement"
)
replace = True
#exclude_index passed to probsample_noreplace needs to be indexed to index2
sampled_index[i, :] = probsample_noreplace(
index2,
sample_size=J,
prob_array=i_prob,
exclude_index=chosen_choice_index_to_index2[i],
return_index=True)
sampling_prob = take(prob, sampled_index)
sampled_index_within_prob = sampled_index.copy()
sampled_index = index2[sampled_index]
is_chosen_choice = zeros(sampled_index.shape, dtype="bool")
#chosen_choice = -1 * ones(chosen_choice_index.size, dtype="int32")
if include_chosen_choice:
sampled_index = column_stack(
(chosen_choice_index[:, newaxis], sampled_index))
is_chosen_choice = zeros(sampled_index.shape, dtype="bool")
is_chosen_choice[chosen_choice_index != UNPLACED_ID, 0] = 1
#chosen_choice[where(is_chosen_choice)[0]] = where(is_chosen_choice)[1]
## this is necessary because prob is indexed to index2, not to the choice set (as is chosen_choice_index)
sampling_prob_for_chosen_choices = take(
prob, chosen_choice_index_to_index2[:, newaxis])
## if chosen choice chosen equals unplaced_id then the sampling prob is 0
sampling_prob_for_chosen_choices[where(
chosen_choice_index == UNPLACED_ID)[0], ] = 0.0
sampling_prob = column_stack(
[sampling_prob_for_chosen_choices, sampling_prob])
interaction_dataset = self.create_interaction_dataset(
dataset1, dataset2, index1, sampled_index)
interaction_dataset.add_attribute(sampling_prob,
'__sampling_probability')
interaction_dataset.add_attribute(is_chosen_choice, 'chosen_choice')
if local_resources.get("include_mnl_bias_correction_term", False):
if include_chosen_choice:
sampled_index_within_prob = column_stack(
(chosen_choice_index_to_index2[:, newaxis],
sampled_index_within_prob))
interaction_dataset.add_mnl_bias_correction_term(
prob, sampled_index_within_prob)
## to get the older returns
#sampled_index = interaction_dataset.get_2d_index()
#chosen_choices = UNPLACED_ID * ones(index1.size, dtype="int32")
#where_chosen = where(interaction_dataset.get_attribute("chosen_choice"))
#chosen_choices[where_chosen[0]]=where_chosen[1]
#return (sampled_index, chosen_choice)
return interaction_dataset
choice_filter_index = None #where(datasets[choice_set_name].get_attribute('zone_id') == 742)[0]
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc','urbansim','opus_core'],
storage=AttributeCache())
choice_set = dataset_pool.get_dataset(choice_set_name)
if choice_filter_index is None:
choice_filter_index = arange(choice_set.size())
specified_coefficients = SpecifiedCoefficients().create(coefficients, specification, neqs=choice_filter_index.size)
interaction_dataset = InteractionDataset(dataset1=agent_set,
dataset2=dataset_pool.get_dataset(choice_set_name),
index1=None,
index2=choice_filter_index)
interaction_dataset.compute_variables(variables,
dataset_pool = dataset_pool)
for index in range(agent_set.size()):
submodel = agent_set.get_attribute('submodel')[index]
if specified_coefficients.nsubmodels == 1:
submodel = -2
coef = SpecifiedCoefficientsFor1Submodel(specified_coefficients,submodel)
data = interaction_dataset.create_logit_data(coef, index=array(index))
coef_names = coef.get_coefficient_names_from_alt().tolist()
ids = reshape(choice_set.get_id_attribute()[choice_filter_index], (1,choice_filter_index.size,1))
write_2d_data(concatenate((ids, data),axis=2), ['id'] + coef_names, filename=os.path.join(file_name_root,
'%s_data_submodel%sa%s_year%s.txt' % (model, submodel, index+1, year)))
def run(self, dataset1, dataset2, index1=None, index2=None, sample_size=10, weight=None,
include_chosen_choice=False, with_replacement=False, resources=None, dataset_pool=None):
"""this function samples number of sample_size (scalar value) alternatives from dataset2
for agent set specified by dataset1.
If index1 is not None, only samples alterantives for agents with indices in index1;
if index2 is not None, only samples alternatives from indices in index2.
sample_size specifies number of alternatives to be sampled for each agent.
weight, to be used as sampling weight, is either an attribute name of dataset2, or a 1d
array of the same length as index2 or 2d array of shape (index1.size, index2.size).
Also refer to document of interaction_dataset"""
if dataset_pool is None:
try:
sc = SessionConfiguration()
dataset_pool=sc.get_dataset_pool()
except:
dataset_pool = DatasetPool()
local_resources = Resources(resources)
local_resources.merge_if_not_None(
{"dataset1": dataset1, "dataset2": dataset2,
"index1":index1, "index2": index2,
"sample_size": sample_size, "weight": weight,
"with_replacement": with_replacement,
"include_chosen_choice": include_chosen_choice})
local_resources.check_obligatory_keys(['dataset1', 'dataset2', 'sample_size'])
agent = local_resources["dataset1"]
index1 = local_resources.get("index1", None)
if index1 is None:
index1 = arange(agent.size())
choice = local_resources["dataset2"]
index2 = local_resources.get("index2", None)
if index2 is None:
index2 = arange(choice.size())
if index1.size == 0 or index2.size == 0:
err_msg = "either choice size or agent size is zero, return None"
logger.log_warning(err_msg)
return None
include_chosen_choice = local_resources.get("include_chosen_choice", False)
J = local_resources["sample_size"]
if include_chosen_choice:
J = J - 1
with_replacement = local_resources.get("with_replacement")
weight = local_resources.get("weight", None)
if isinstance(weight, str):
if weight in choice.get_known_attribute_names():
weight=choice.get_attribute(weight)
rank_of_weight = 1
elif VariableName(weight).get_dataset_name() == choice.get_dataset_name():
weight=choice.compute_variables(weight, dataset_pool=dataset_pool)
rank_of_weight = 1
else:
## weights can be an interaction variable
interaction_dataset = InteractionDataset(local_resources)
weight=interaction_dataset.compute_variables(weight, dataset_pool=dataset_pool)
rank_of_weight = 2
elif isinstance(weight, ndarray):
rank_of_weight = weight.ndim
elif not weight: ## weight is None or empty string
weight = ones(index2.size)
rank_of_weight = 1
else:
err_msg = "unkown weight type"
logger.log_error(err_msg)
raise TypeError, err_msg
if (weight.size <> index2.size) and (weight.shape[rank_of_weight-1] <> index2.size):
if weight.shape[rank_of_weight-1] == choice.size():
if rank_of_weight == 1:
weight = take(weight, index2)
if rank_of_weight == 2:
weight = take(weight, index2, axis=1)
else:
err_msg = "weight array size doesn't match to size of dataset2 or its index"
logger.log_error(err_msg)
raise ValueError, err_msg
prob = normalize(weight)
#chosen_choice = ones(index1.size) * UNPLACED_ID
chosen_choice_id = agent.get_attribute(choice.get_id_name()[0])[index1]
#index_of_placed_agent = where(greater(chosen_choice_id, UNPLACED_ID))[0]
chosen_choice_index = choice.try_get_id_index(chosen_choice_id, return_value_if_not_found=UNPLACED_ID)
chosen_choice_index_to_index2 = lookup(chosen_choice_index, index2, index_if_not_found=UNPLACED_ID)
if rank_of_weight == 1: # if weight_array is 1d, then each agent shares the same weight for choices
replace = with_replacement # sampling with no replacement
if nonzerocounts(weight) < J:
logger.log_warning("weight array dosen't have enough non-zero counts, use sample with replacement")
replace = True
sampled_index = prob2dsample( index2, sample_size=(index1.size, J),
prob_array=prob, exclude_index=chosen_choice_index_to_index2,
replace=replace, return_index=True )
#return index2[sampled_index]
if rank_of_weight == 2:
sampled_index = zeros((index1.size,J), dtype="int32") - 1
for i in range(index1.size):
replace = with_replacement # sampling with/without replacement
i_prob = prob[i,:]
if nonzerocounts(i_prob) < J:
logger.log_warning("weight array dosen't have enough non-zero counts, use sample with replacement")
replace = True
#exclude_index passed to probsample_noreplace needs to be indexed to index2
sampled_index[i,:] = probsample_noreplace( index2, sample_size=J, prob_array=i_prob,
exclude_index=chosen_choice_index_to_index2[i],
return_index=True )
sampling_prob = take(prob, sampled_index)
sampled_index = index2[sampled_index]
is_chosen_choice = zeros(sampled_index.shape, dtype="bool")
#chosen_choice = -1 * ones(chosen_choice_index.size, dtype="int32")
if include_chosen_choice:
sampled_index = column_stack((chosen_choice_index[:,newaxis],sampled_index))
is_chosen_choice = zeros(sampled_index.shape, dtype="bool")
is_chosen_choice[chosen_choice_index!=UNPLACED_ID, 0] = 1
#chosen_choice[where(is_chosen_choice)[0]] = where(is_chosen_choice)[1]
## this is necessary because prob is indexed to index2, not to the choice set (as is chosen_choice_index)
sampling_prob_for_chosen_choices = take(prob, chosen_choice_index_to_index2[:, newaxis])
## if chosen choice chosen equals unplaced_id then the sampling prob is 0
sampling_prob_for_chosen_choices[where(chosen_choice_index==UNPLACED_ID)[0],] = 0.0
sampling_prob = column_stack([sampling_prob_for_chosen_choices, sampling_prob])
interaction_dataset = self.create_interaction_dataset(dataset1, dataset2, index1, sampled_index)
interaction_dataset.add_attribute(sampling_prob, '__sampling_probability')
interaction_dataset.add_attribute(is_chosen_choice, 'chosen_choice')
## to get the older returns
#sampled_index = interaction_dataset.get_2d_index()
#chosen_choices = UNPLACED_ID * ones(index1.size, dtype="int32")
#where_chosen = where(interaction_dataset.get_attribute("chosen_choice"))
#chosen_choices[where_chosen[0]]=where_chosen[1]
#return (sampled_index, chosen_choice)
return interaction_dataset
for year in years:
SimulationState().set_current_time(year)
SessionConfiguration().get_dataset_pool().remove_all_datasets()
dataset_pool = DatasetPool(
package_order=['psrc', 'urbansim', 'opus_core'],
storage=AttributeCache())
choice_set = dataset_pool.get_dataset(choice_set_name)
if choice_filter_index is None:
choice_filter_index = arange(choice_set.size())
specified_coefficients = SpecifiedCoefficients().create(
coefficients, specification, neqs=choice_filter_index.size)
interaction_dataset = InteractionDataset(
dataset1=agent_set,
dataset2=dataset_pool.get_dataset(choice_set_name),
index1=None,
index2=choice_filter_index)
interaction_dataset.compute_variables(variables,
dataset_pool=dataset_pool)
for index in range(agent_set.size()):
submodel = agent_set.get_attribute('submodel')[index]
if specified_coefficients.nsubmodels == 1:
submodel = -2
coef = SpecifiedCoefficientsFor1Submodel(specified_coefficients,
submodel)
data = interaction_dataset.create_logit_data(coef,
index=array(index))
coef_names = coef.get_coefficient_names_from_alt().tolist()
data=[85,20,0,90,35,51,0,10,5])
from opus_core.datasets.dataset_pool import DatasetPool
dataset_pool = DatasetPool(package_order=['urbansim', 'opus_core'], storage=storage)
dataset_pool.datasets_in_pool()
hs = dataset_pool.get_dataset("household")
dataset_pool.datasets_in_pool()
hs.size()
constant = dataset_pool.get_dataset("urbansim_constant")
constant["percent_coverage_threshold"] = 50
locations.compute_variables(["urbansim.gridcell.is_in_wetland"], dataset_pool=dataset_pool)
locations.get_attribute("is_in_wetland")
# Interaction variables
from opus_core.datasets.interaction_dataset import InteractionDataset
interactions = InteractionDataset(dataset1 = households, dataset2 = locations)
interactions.get_dataset_name()
from numpy import arange
interactions = InteractionDataset(dataset1 = households, dataset2 = locations, index1 = arange(5), index2 = arange(3))
interactions.compute_variables(["urbansim.household_x_gridcell.cost_times_income"])
specification = EquationSpecification(
variables=array([
"gridcell.cost",
"urbansim.household_x_gridcell.cost_times_income"]),
coefficients=array(["costcoef", "cti_coef"]))
households.add_primary_attribute(data=[2,8,3,1,5,4,9,7,3,6], name="location")
coef, other_results = hlcm.estimate(specification, households)
# Versioning
