def test_ones_like(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"attr1": array([5, 10, 0, 3]),
"attr2": array([5.0, 0.0, 8.0, 1.1]),
"id": arange(4)
})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
expr1 = 'ones_like(attr1)'
result = dataset.compute_variables([expr1])
should_be = array([1, 1, 1, 1])
self.assert_(numpy.allclose(result, array(should_be), rtol=1e-6),
"Error in ones_like")
self.assert_(result.dtype == should_be.dtype, "Error in ones_like")
expr2 = 'ones_like(attr1)'
result = dataset.compute_variables([expr2])
should_be = array([1, 1, 1, 1])
self.assert_(numpy.allclose(result, array(should_be), rtol=1e-6),
"Error in ones_like")
self.assertEqual(result.dtype, should_be.dtype, "Error in ones_like")
def test_disaggregate(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id': array([1, 2, 3, 4]),
'id2': array([1, 2, 1, 2])
})
storage.write_table(table_name='faz',
table_data={
'my_variable': array([4, 8]),
'id2': array([1, 2])
})
ds = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id",
dataset_name="myzone")
ds2 = Dataset(in_storage=storage,
in_table_name='faz',
id_name="id2",
dataset_name="myfaz")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds)
dataset_pool._add_dataset('myfaz', ds2)
values = ds.compute_variables(
["myzone.disaggregate(myfaz.my_variable)"],
dataset_pool=dataset_pool)
should_be = array([4, 8, 4, 8])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in disaggregate")
def test_alias_complex_expression(self):
# aliasing a complex expression
expr = "x = 2*sqrt(var1+var2)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"var1": array([4, -8, 0.5, 1]),
"var2": array([3, 3, 7, 7]),
"id": array([1, 2, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([5.29150262, 0.0, 5.47722558, 5.65685425])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_alias_complex_expression")
# check that the new var has x as an alias
v = VariableName(expr)
self.assertEqual(v.get_alias(), 'x', msg="bad value for alias")
# check that the alias gives the correct value
result2 = dataset.compute_variables(['x'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6),
"Error in accessing a_test_variable")
def test_unary_functions_fully_qualified_name(self):
# this tests expressions with unary functions applied to a fully qualified name
expr = "sqrt(opus_core.tests.a_test_variable)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"a_dependent_variable":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-3), True, msg="error in test_unary_functions_fully_qualified_name")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable")
self.assertEqual(name.get_package_name(), None, msg="bad value for package")
self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset")
self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname")
self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias")
# make an instance of the class and check the dependencies (since the dependent variables
# all have fully-qualifed names we don't need to associate a dataset with the variable
# for this test)
self.assertEqual(autogen().dependencies(), ['opus_core.tests.a_test_variable'],
msg="dependencies are incorrect")
def test_aggregate_sum(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'my_variable': array([4, 8, 0.5, 1]),
'id': array([1, 2, 3, 4]),
'id2': array([1, 2, 1, 2]),
})
storage.write_table(table_name='faz',
table_data={"id2": array([1, 2])})
ds = Dataset(in_storage=storage,
in_table_name='zones',
id_name="id",
dataset_name="myzone")
ds2 = Dataset(in_storage=storage,
in_table_name='faz',
id_name="id2",
dataset_name="myfaz")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds)
dataset_pool._add_dataset('myfaz', ds2)
values = ds2.compute_variables(
['myfaz.aggregate(10.0*myzone.my_variable, function=sum)'],
dataset_pool=dataset_pool)
should_be = array([45, 90])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in aggregate_sum")
class SimpleModelTest(opus_unittest.OpusTestCase):
def setUp(self):
self.data = {
'id':
arange(10) + 1,
'attribute':
array([3000, 2800, 1000, 550, 600, 1000, 2000, 500, 100, 1000])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset', table_data=self.data)
self.dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name=['id'])
def test_simple_model(self):
m = SimpleModel()
m.run(self.dataset,
'sqrt(dataset.attribute)',
outcome_attribute='sqrtattr')
self.assertEqual(
ma.allclose(self.dataset.get_attribute('sqrtattr'),
sqrt(self.data['attribute'])), True)
self.assertEqual(
'sqrtattr' in self.dataset.get_primary_attribute_names(), True)
def test_simple_model_without_outcome_attribute(self):
m = SimpleModel()
m.run(self.dataset, 'lattr = ln(dataset.attribute)')
self.assertEqual(
ma.allclose(self.dataset.get_attribute('lattr'),
log(self.data['attribute'])), True)
self.assertEqual('lattr' in self.dataset.get_primary_attribute_names(),
True)
def test_fully_qualified_variable(self):
# this tests an expression consisting of a fully-qualified variable
expr = "opus_core.test_agent.income_times_2"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="test_agents", table_data={"income": array([1, 5, 10]), "id": array([1, 3, 4])})
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([2, 10, 20])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_variable")
# check that expr is in the cache of known expressions
# (normally we shouldn't be accessing this private field, but just this once ...)
cache = VariableName._cache
self.assert_(expr in cache, msg="did not find expr in cache")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
self.assertEqual(name.get_package_name(), "opus_core", msg="bad value for package")
self.assertEqual(name.get_dataset_name(), "test_agent", msg="bad value for dataset")
self.assertEqual(name.get_short_name(), "income_times_2", msg="bad value for shortname")
self.assertEqual(name.get_alias(), "income_times_2", msg="bad value for alias")
self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
# test that the variable can now also be accessed using its short name in an expression
result2 = dataset.compute_variables(["income_times_2"])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable")
# check that the cache uses the variable name with whitespace removed
oldsize = len(cache)
expr_with_spaces = "opus_core . test_agent. income_times_2 "
name2 = VariableName(expr_with_spaces)
newsize = len(cache)
self.assertEqual(oldsize, newsize, msg="caching error")
self.assert_(expr_with_spaces not in cache, msg="caching error")
self.assertEqual(expr_with_spaces, name2.get_expression(), msg="caching error")
self.assertEqual(name2.get_short_name(), "income_times_2", msg="bad value for shortname")
def test_constants(self):
# test an expression involving two dataset names, one of which is *_constant
expr = "test_agent.age<=opus_constant.young_age"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='test_agents',
table_data={
"age":array([30,20,60,80]),
"id":array([1,3,4,10])
}
)
storage.write_table(
table_name='opus_constants',
table_data={
"young_age":array([35]),
"opus_constant_id":array([1])
}
)
dataset_pool = DatasetPool(storage=storage)
# Test that the dataset name is correct for expr. It should be test_agent -- opus_constant just holds constants,
# and is ignored as far as finding the dataset name for the expression.
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assertEqual(name.get_package_name(), None)
self.assertEqual(name.get_dataset_name(), 'test_agent')
# make an instance of the class and check the dependencies (it shouldn't depend on urbansim_constant)
self.assertEqual(autogen().dependencies(), ['test_agent.age'])
dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr], dataset_pool=dataset_pool)
should_be = array( [True,True,False,False] )
self.assertEqual( ma.allequal( result, should_be), True)
def test_fully_qualified_DDD_SSS_variable(self):
# this should use the test variable a_test_SSS_variable_DDD_SSS
expr = "opus_core.tests.a_test_squid_variable_42_clam"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='tests',
table_data={
"a_dependent_variable":array([1,5,10]),
"id":array([1,3,4])
}
)
dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([10,50,100])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_DDD_SSS_variable")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package")
self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset")
self.assertEqual(name.get_short_name(), 'a_test_squid_variable_42_clam', msg="bad value for shortname")
self.assertEqual(name.get_alias(), 'a_test_squid_variable_42_clam', msg="bad value for alias")
self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
# test that the variable can now also be accessed using its short name in an expression
result2 = dataset.compute_variables(['a_test_squid_variable_42_clam'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_squid_variable_42_clam")
def test_number_of_agents_expression(self):
expr = "mygridcell.number_of_agents(myjob)+10"
storage = StorageFactory().get_storage('dict_storage')
gridcell_grid_id = array([1, 2, 3])
job_grid_id = array(
[2, 1, 3, 1]
) #specify an array of 4 jobs, 1st job's grid_id = 2 (it's in gridcell 2), etc.
storage.write_table(table_name='gridcells',
table_data={'gid': gridcell_grid_id})
storage.write_table(table_name='jobs',
table_data={
'jid': arange(4) + 1,
'gid': job_grid_id
})
gs = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="gid",
dataset_name="mygridcell")
jobs = Dataset(in_storage=storage,
in_table_name='jobs',
id_name="jid",
dataset_name="myjob")
values = gs.compute_variables([expr],
resources=Resources({
"myjob": jobs,
"mygridcell": gs
}))
should_be = array([12, 11, 11])
self.assert_(ma.allclose(values, should_be, rtol=1e-7),
msg="Error in " + expr)
def test_casts_fully_qualified_variable(self):
expr1 = "opus_core.test_agent.income_times_10.astype(int32)"
expr2 = "opus_core.test_agent.income_times_10.astype(int32)**2"
expr3 = "(2*opus_core.test_agent.income_times_10).astype(int32)"
error_msg = "Error in test_casts_fully_qualified_variable"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='test_agents',
table_data={
"income": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='test_agents',
id_name="id",
dataset_name="test_agent")
result1 = dataset.compute_variables([expr1])
self.assertEqual(type(result1[0]), int32, error_msg)
self.assert_(ma.allclose(result1, array([10, 50, 100]), rtol=1e-6),
error_msg)
result2 = dataset.compute_variables([expr2])
self.assertEqual(type(result2[0]), int32, error_msg)
self.assert_(
ma.allclose(result2, array([100, 2500, 10000]), rtol=1e-6),
error_msg)
result3 = dataset.compute_variables([expr3])
self.assertEqual(type(result3[0]), int32, error_msg)
self.assert_(ma.allclose(result3, array([20, 100, 200]), rtol=1e-6),
error_msg)
def test_expression(self):
# test an expression. Also make sure that the generated variable can be accessued
# using its short name and that dependencies are correct.
expr = "2*sqrt(my_variable+10)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"my_variable": array([4, -8, 0.5, 1]),
"id": array([1, 2, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([7.48331477, 2.82842712, 6.4807407, 6.63324958])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_expression")
# check the name
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
self.assertEqual(var.name(), expr, msg="name is incorrect")
# check the dependencies
self.assertEqual(var.dependencies(), ['mydataset.my_variable'],
msg="dependencies are incorrect")
# test that the variable can now also be accessed using its short name in an expression
result2 = dataset.compute_variables([v.get_short_name()])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6),
"Error in accessing a_test_variable")
def test_expression_2vars(self):
# test an expression with 2 variables
expr = "2*sqrt(var1+var2)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset',
table_data={
"var1": array([4, -8, 0.5, 1]),
"var2": array([3, 3, 7, 7]),
"id": array([1, 2, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name="id",
dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([5.29150262, 0.0, 5.47722558, 5.65685425])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_expression_2vars")
# check the dependencies (will depend on two different other variables)
v = VariableName(expr)
var = VariableFactory().get_variable(v, dataset)
# use sets for the equality test, since we don't know in what order the dependencies will be returned
self.assertEqual(set(var.dependencies()),
set(['mydataset.var1', 'mydataset.var2']),
msg="dependencies are incorrect")
def test_change_three_elements(self):
"""3 values are in common - change them to -1. Other attributes stay unchanged."""
data = {
'my_id': array([1,2,3,4,5]),
'attr': array([10,2,3,50,2]),
'attr2': array([4,3,2,5,3])
}
data2 = {
'attr': array([2,6,7,3])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset', table_data=data)
dataset = Dataset(in_storage = storage,
in_table_name='dataset',
id_name='my_id'
)
storage.write_table(table_name='dataset2', table_data=data2)
dataset2 = Dataset(in_storage = storage,
in_table_name='dataset2',
id_name='attr'
)
JoinAttributeModificationModel().run(dataset,dataset2, value=-1)
self.assertEqual(ma.allequal(dataset.get_attribute('attr'), array([10,-1,-1,50,-1])), True)
self.assertEqual(ma.allequal(dataset.get_attribute('attr2'), data['attr2']), True)
def test_disaggregate_fully_qualified_variable(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id': array([1, 2, 3, 4]),
'id': array([1, 2, 1, 2])
})
# it would be nicer to call this table 'fazzes' but we want to use the existing test variable
storage.write_table(table_name='test_locations',
table_data={
'cost': array([4, 8]),
'id': array([1, 2])
})
zone_dataset = Dataset(in_storage=storage,
in_table_name='zones',
id_name="zone_id",
dataset_name="zone")
test_dataset = Dataset(in_storage=storage,
in_table_name='test_locations',
id_name="id",
dataset_name='test_location')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('zone', zone_dataset)
dataset_pool._add_dataset('test_location', test_dataset)
values = zone_dataset.compute_variables(
['zone.disaggregate(opus_core.test_location.cost_times_3)'],
dataset_pool=dataset_pool)
should_be = array([12, 24, 12, 24])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in test_disaggregate_fully_qualified_variable")
def test_join_by_rows_for_unique_ids(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset1',
table_data={
"id":array([2,4]),
"attr":array([4,7])
}
)
storage.write_table(
table_name='dataset2',
table_data={
"id":array([1,2]),
"attr":array([55,66])
}
)
ds1 = Dataset(in_storage=storage, in_table_name='dataset1', id_name='id')
ds2 = Dataset(in_storage=storage, in_table_name='dataset2', id_name='id')
threw_exception = False
try:
ds1.join_by_rows(ds2)
except StandardError:
threw_exception = True
self.assert_(threw_exception)
def test_aggregate(self):
# test aggregate with no function specified (so defaults to 'sum')
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id': array([1, 2]),
})
storage.write_table(table_name='gridcells',
table_data={
'my_variable': array([4, 8, 0.5, 1]),
'grid_id': array([1, 2, 3, 4]),
'zone_id': array([1, 2, 1, 2]),
})
zone_dataset = Dataset(in_storage=storage,
in_table_name='zones',
id_name="zone_id",
dataset_name='zone')
gridcell_dataset = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="grid_id",
dataset_name='gridcell')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('gridcell', gridcell_dataset)
dataset_pool._add_dataset('zone', zone_dataset)
values = zone_dataset.compute_variables(
['zone.aggregate(gridcell.my_variable)'],
dataset_pool=dataset_pool)
should_be = array([4.5, 9])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in aggregate")
def get_households_for_estimation(agent_set, in_storage,
agents_for_estimation_table_name,
exclude_condition=None,
join_datasets=True):
estimation_set = Dataset(in_storage = in_storage,
in_table_name=agents_for_estimation_table_name,
id_name=agent_set.get_id_name(),
dataset_name=agent_set.get_dataset_name())
agent_set.unload_primary_attributes()
agent_set.load_dataset(attributes='*')
estimation_set.load_dataset(attributes=agent_set.get_primary_attribute_names())
if join_datasets:
agent_set.join_by_rows(estimation_set,
require_all_attributes=False,
change_ids_if_not_unique=True)
index = arange(agent_set.size()-estimation_set.size(),agent_set.size())
else:
index = agent_set.get_id_index(estimation_set.get_id_attribute())
exclude_ids = []
if exclude_condition is not None:
exclude_ids = agent_set.get_id_attribute()[where(agent_set.compute_variables(exclude_condition))]
for id in exclude_ids:
minus = agent_set.get_id_index(id)
if minus in index:
index = index[index != minus]
return (agent_set, index)
def cache_database_table(self, table_name, base_year, database, in_storage, config):
"""Copy this table from input database into attribute cache.
"""
logger.start_block('Caching table %s' % table_name)
try:
#TODO: why is the config being modified...seems like its kind of useless here...
config['storage_location'] = os.path.join(config['cache_directory'], str(base_year), table_name)
if not os.path.exists(config['storage_location']):
flt_storage = StorageFactory().get_storage(
type='flt_storage',
subdir='store',
storage_location=config['storage_location'])
table = database.get_table(table_name)
id_name = [primary_key.name.lower() for primary_key in table.primary_key]
dataset = Dataset(resources=config,
in_storage=in_storage,
out_storage=flt_storage,
in_table_name=table_name,
id_name = id_name)
nchunks = config['creating_baseyear_cache_configuration'].tables_to_cache_nchunks.get(table_name, 1)
current_time = SimulationState().get_current_time()
SimulationState().set_current_time(base_year)
dataset.load_dataset(nchunks=nchunks, flush_after_each_chunk=True)
SimulationState().set_current_time(current_time)
else:
logger.log_status(config['storage_location'] + " already exits; skip caching " + table_name)
finally:
logger.end_block()
def __init__(self, resources=None, in_storage=None, out_storage=None,
in_table_name=None, attributes=None,
out_table_name=None, id_name=None,
nchunks=None, other_in_table_names=None,
debuglevel=0):
debug = DebugPrinter(debuglevel)
debug.print_debug("Creating DevelopmentGroupDataset object.",2)
resources = ResourceCreatorDevelopmentGroups().get_resources_for_dataset(
resources = resources,
in_storage = in_storage,
out_storage = out_storage,
in_table_name = in_table_name,
out_table_name = out_table_name,
attributes = attributes,
id_name = id_name,
id_name_default = self.id_name_default,
nchunks = nchunks,
debug = debug
)
Dataset.__init__(self,resources = resources)
if isinstance(other_in_table_names,list):
for place_name in other_in_table_names: #load other tables
ds = Dataset(resources = resources)
ds.load_dataset(in_table_name=place_name)
self.connect_datasets(ds)
def test_aggregate_unqualified_name(self):
# test aggregate without the dataset provided for the variable being aggregated
expr = 'zone.aggregate(my_variable)'
# to be correct, should be 'zone.aggregate(gridcell.my_variable)'
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id': array([1, 2]),
})
storage.write_table(table_name='gridcells',
table_data={
'my_variable': array([4, 8, 0.5, 1]),
'grid_id': array([1, 2, 3, 4]),
'zone_id': array([1, 2, 1, 2]),
})
zone_dataset = Dataset(in_storage=storage,
in_table_name='zones',
id_name="zone_id",
dataset_name='zone')
gridcell_dataset = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="grid_id",
dataset_name='gridcell')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('gridcell', gridcell_dataset)
dataset_pool._add_dataset('zone', zone_dataset)
self.assertRaises(ValueError,
zone_dataset.compute_variables, [expr],
dataset_pool=dataset_pool)
def test_change_with_index_and_filter(self):
"""The secondary dataset is restricted by index and filter."""
data = {
'my_id': array([1,2,3,4,5,6]),
'attr': array([10,20,30,50,46,100]),
'attr2': array(6*[1])
}
data2 = {
'attr': array([20, 6, 7, 3, 10, 30, 100, 50])
}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset', table_data=data)
dataset = Dataset(in_storage = storage,
in_table_name='dataset',
id_name='my_id'
)
storage.write_table(table_name='dataset2', table_data=data2)
dataset2 = Dataset(in_storage = storage,
in_table_name='dataset2',
id_name='attr'
)
JoinAttributeModificationModel().run(dataset,dataset2, index=array([0,1,2,7]), attribute_to_be_modified='attr2',
filter='attr > 20')
self.assertEqual(ma.allequal(dataset.get_attribute('attr2'), array([1, 1, 1, 0, 1, 1])), True)
def test_alias_fully_qualified_variable(self):
expr = "x = opus_core.tests.a_test_variable"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"a_dependent_variable": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([10, 50, 100])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_alias_fully_qualified_variable")
# check that the new var has x as an alias
v = VariableName(expr)
self.assertEqual(v.get_package_name(),
None,
msg="bad value for package_name")
self.assertEqual(v.get_dataset_name(),
'tests',
msg="bad value for dataset_name")
self.assert_(v.get_short_name().startswith('autogen'),
msg="bad value for shortname")
self.assertEqual(v.get_alias(), 'x', msg="bad value for alias")
self.assertNotEqual(v.get_autogen_class(),
None,
msg="bad value for autogen_class")
# check that the alias has the correct value
result2 = dataset.compute_variables(['x'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6),
"Error in accessing a_test_variable")
def test_estimation_with_restricted_submodel_size(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='dataset',
table_data={
"id":array([1,2,3,4,5,6,7,8,9,10]),
"attr1":array([4,7,2,1,5,4,5,3,2,1]),
"attr2":array([6.8,2.6,0,1,0,4.3,2.1,6,8,7,]),
"submodel_id": array([1,2,2,1,1,1,2,1,1,1]),
"outcome": array([0,1,0,1,1,1,0,0,1,1])
}
)
ds = Dataset(in_storage=storage, in_table_name='dataset', id_name="id")
specification_2subm = EquationSpecification(
variables=array(["constant", "attr2", "constant", "attr1"]),
coefficients=array(["constant", "ba2", "constant", "ba1"]),
submodels = array([1,1,2,2]))
model = RegressionModel(submodel_string="submodel_id", estimate_config={'submodel_size_max': 5})
model.estimate(specification_2subm, ds, "outcome", procedure='opus_core.estimate_linear_regression')
data_attr1 = model.get_data("ba1", 1)
self.assert_(data_attr1 == None, msg = "Error in getting data from regression model with multiple submodels.")
data_attr1 = model.get_data("ba2", 1)
self.assert_(data_attr1.size == 5, msg = "Error in sub-sampling data in regression model.")
data_attr1 = model.get_data("ba1", 2)
self.assert_(ma.allequal(ds.get_attribute("attr1")[array([1, 2, 6])], data_attr1),
msg = "Error in getting data from regression model with multiple submodels.")
def __init__(self,
resources=None,
in_storage=None,
out_storage=None,
in_table_name=None,
out_table_name=None,
attributes=None,
id_name=None,
nchunks=None,
debuglevel=0
):
try:
debug = SessionConfiguration().get('debuglevel', 0)
except:
debug = 0
debug = DebugPrinter(debug)
if debuglevel > debug.flag:
debug.flag = debuglevel
debug.print_debug("Creating object %s.%s" % (self.__class__.__module__, self.__class__.__name__), 2)
resources = ResourceFactory().get_resources_for_dataset(
self.dataset_name,
resources = resources,
in_storage = in_storage,
in_table_name_pair = (in_table_name,self.in_table_name_default),
attributes_pair = (attributes,self.attributes_default),
out_storage = out_storage,
out_table_name_pair = (out_table_name, self.out_table_name_default),
id_name_pair = (id_name, self.id_name_default),
nchunks_pair = (nchunks,self.nchunks_default),
debug_pair = (debug,None),
)
CoreDataset.__init__(self,resources = resources)
def test_disaggregate_one_level(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id0':arange(7)+1,
'id1':array([1,3,1,2,3,2,1])
}
)
storage.write_table(table_name='fazes',
table_data={
'id1':array([1,2,3]),
'id2':array([1,2,1])
}
)
storage.write_table(table_name='fazdistr',
table_data={
'my_variable':array([40,50]),
'id2':array([1,2])
}
)
ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone")
ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz")
ds2 = Dataset(in_storage=storage, in_table_name='fazdistr', id_name="id2", dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds0.compute_variables(["myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"], dataset_pool=dataset_pool)
should_be = array([40, 40, 40, 50, 40,50, 40])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_one_level")
def test_disaggregate_and_multiply(self):
# Perform two different disaggregations and multiply the results. This tests using a dataset name in both the
# list of intermediates and as the dataset being disaggregated (myfaz in this case).
expr = "myzone.disaggregate(myfaz.fazsqft) * myzone.disaggregate(myfazdistr.my_variable, intermediates=[myfaz])"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'id0':arange(7)+1,
'id1':array([1,3,1,2,3,2,1])
}
)
storage.write_table(table_name='fazes',
table_data={
'id1':array([1,2,3]),
'id2':array([1,2,1]),
'fazsqft':array([10,50,100])
}
)
storage.write_table(table_name='fazdistrs',
table_data={
'my_variable':array([40,50]),
'id2':array([1,2])
}
)
ds0 = Dataset(in_storage=storage, in_table_name='zones', id_name="id0", dataset_name="myzone")
ds1 = Dataset(in_storage=storage, in_table_name='fazes', id_name="id1", dataset_name="myfaz")
ds2 = Dataset(in_storage=storage, in_table_name='fazdistrs', id_name="id2", dataset_name="myfazdistr")
dataset_pool = DatasetPool()
dataset_pool._add_dataset('myzone', ds0)
dataset_pool._add_dataset('myfaz', ds1)
dataset_pool._add_dataset('myfazdistr', ds2)
values = ds0.compute_variables([expr], dataset_pool=dataset_pool)
should_be = array([400, 4000, 400, 2500, 4000, 2500, 400])
self.assert_(ma.allclose(values, should_be, rtol=1e-6), "Error in disaggregate_and_multiply")
def test_agent_times_choice(self):
expression = 'agent_x_choice.agent_times_choice(attr)'
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='agents',
table_data={'id': array([1, 2, 3, 4, 5]), 'attr_2': array([3, 2, 4, 10, 20]),
'attr_3': array([10, 100, 1000, 500, 0]),
'attr_4': array([100, 500, 0, 20, -30])
}
)
storage.write_table(table_name='choices',
table_data={'id': array([1, 2, 3, 4])}
)
agents = Dataset(in_storage=storage, in_table_name='agents', dataset_name='agent', id_name='id')
choices = Dataset(in_storage=storage, in_table_name='choices', dataset_name='choice', id_name='id')
ids = InteractionDataset(dataset1=agents, dataset2=choices, index1=array([0,1,3,4]), index2=array([1,2,3]))
result = ids.compute_variables(expression)
should_be = array([[3, 10, 100], [2,100,500], [10,500, 20], [20, 0, -30]])
self.assertEqual(ma.allequal(result, should_be), True)
agents.touch_attribute('attr_2') # in order to recompute the expression
choices.add_primary_attribute(name='name', data=array(['bus', 'car', 'tran', 'walk']))
agents.add_primary_attribute(name='attr_tran', data=array([100, 1000, 10000, 5000,10]))
result = ids.compute_variables(expression)
should_be = array([[3, 100, 100], [2,1000,500], [10,5000, 20], [20, 10, -30]])
self.assertEqual(ma.allequal(result, should_be), True)
def __init__(self, *args, **kwargs):
Dataset.__init__(self,
dataset_name="test_location",
id_name="id",
in_table_name=kwargs.get('in_table_name',
'test_locations'),
in_storage=kwargs['in_storage'])
def test_constants(self):
# test an expression involving two dataset names, one of which is *_constant
expr = "test_agent.age<=opus_constant.young_age"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='test_agents',
table_data={
"age": array([30, 20, 60, 80]),
"id": array([1, 3, 4, 10])
})
storage.write_table(table_name='opus_constants',
table_data={
"young_age": array([35]),
"opus_constant_id": array([1])
})
dataset_pool = DatasetPool(storage=storage)
# Test that the dataset name is correct for expr. It should be test_agent -- opus_constant just holds constants,
# and is ignored as far as finding the dataset name for the expression.
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assertEqual(name.get_package_name(), None)
self.assertEqual(name.get_dataset_name(), 'test_agent')
# make an instance of the class and check the dependencies (it shouldn't depend on urbansim_constant)
self.assertEqual(autogen().dependencies(), ['test_agent.age'])
dataset = Dataset(in_storage=storage,
in_table_name='test_agents',
id_name="id",
dataset_name="test_agent")
result = dataset.compute_variables([expr], dataset_pool=dataset_pool)
should_be = array([True, True, False, False])
self.assertEqual(ma.allequal(result, should_be), True)
def test_alias_attribute_same_name(self):
# this tests an expression consisting of an alias for a primary attribute that is the same name as the primary attribute
expr = "persons = persons"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"persons": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result = dataset.compute_variables([expr])
self.assertEqual(ma.allclose(result, [1, 5, 10], rtol=1e-7),
True,
msg="error in test_alias_attribute")
name = VariableName(expr)
self.assertEqual(name.get_short_name(),
'persons',
msg="bad value for shortname")
self.assertEqual(name.get_alias(),
'persons',
msg="bad value for alias")
self.assertEqual(name.get_autogen_class(),
None,
msg="bad value for autogen_class")
def test_alias_fully_qualified_variable_same_name(self):
expr = "a_test_variable = opus_core.tests.a_test_variable"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"a_dependent_variable": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([10, 50, 100])
self.assert_(ma.allclose(result, should_be, rtol=1e-6),
"Error in test_alias_fully_qualified_variable")
result2 = dataset.compute_variables(['a_test_variable'])
self.assert_(ma.allclose(result2, should_be, rtol=1e-6),
"Error in accessing a_test_variable")
v = VariableName(expr)
# check that no autogen class was generated
self.assertEqual(v.get_autogen_class(),
None,
msg="bad value for autogen_class")
# check that the alias is correct
self.assertEqual(v.get_alias(),
'a_test_variable',
msg="bad value for alias")
def test_change_with_index_and_filter(self):
"""The secondary dataset is restricted by index and filter."""
data = {
'my_id': array([1, 2, 3, 4, 5, 6]),
'attr': array([10, 20, 30, 50, 46, 100]),
'attr2': array(6 * [1])
}
data2 = {'attr': array([20, 6, 7, 3, 10, 30, 100, 50])}
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='dataset', table_data=data)
dataset = Dataset(in_storage=storage,
in_table_name='dataset',
id_name='my_id')
storage.write_table(table_name='dataset2', table_data=data2)
dataset2 = Dataset(in_storage=storage,
in_table_name='dataset2',
id_name='attr')
JoinAttributeModificationModel().run(dataset,
dataset2,
index=array([0, 1, 2, 7]),
attribute_to_be_modified='attr2',
filter='attr > 20')
self.assertEqual(
ma.allequal(dataset.get_attribute('attr2'),
array([1, 1, 1, 0, 1, 1])), True)
def test_aggregate_squared_with_cast(self):
# more exercising the SUBPATTERN_NUMBER_OF_AGENTS_WITH_CAST tree pattern
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id': array([1, 2]),
})
storage.write_table(table_name='gridcells',
table_data={
'my_variable': array([4, 8, 0.5, 1]),
'grid_id': array([1, 2, 3, 4]),
'zone_id': array([1, 2, 1, 2]),
})
zone_dataset = Dataset(in_storage=storage,
in_table_name='zones',
id_name="zone_id",
dataset_name='zone')
gridcell_dataset = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="grid_id",
dataset_name='gridcell')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('gridcell', gridcell_dataset)
dataset_pool._add_dataset('zone', zone_dataset)
values = zone_dataset.compute_variables(
['(zone.aggregate(gridcell.my_variable)**2).astype(float32)'],
dataset_pool=dataset_pool)
should_be = array([4.5 * 4.5, 9.0 * 9.0])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in aggregate")
def test_alias_file(self):
# this tests aliases in the file 'aliases.py'
# expr1 and expr2 are aliases, while expr3 is an ordinary variable,
# just to make sure that aliases and ordinary variables interoperate correctly
expr1 = "opus_core.test_agent.income_times_10"
expr2 = "opus_core.test_agent.income_times_5"
expr3 = "opus_core.test_agent.income_times_2"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='test_agents',
table_data={
"income": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='test_agents',
id_name="id",
dataset_name="test_agent")
result1 = dataset.compute_variables([expr1])
self.assert_(ma.allclose(result1, array([10, 50, 100]), rtol=1e-6),
"Error in test_alias_file")
result2 = dataset.compute_variables([expr2])
self.assert_(ma.allclose(result2, array([5, 25, 50]), rtol=1e-6),
"Error in test_alias_file")
result3 = dataset.compute_variables([expr3])
self.assert_(ma.allclose(result3, array([2, 10, 20]), rtol=1e-6),
"Error in test_alias_file")
def test_aggregate_fully_qualified_variable(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id': array([1, 2]),
})
# it would be nicer to call this table 'gridcells' but we want to use the existing test variable
storage.write_table(table_name='tests',
table_data={
'a_dependent_variable': array([4, 8, 0.5, 1]),
'id': array([1, 2, 3, 4]),
'zone_id': array([1, 2, 1, 2]),
})
zone_dataset = Dataset(in_storage=storage,
in_table_name='zones',
id_name="zone_id",
dataset_name='zone')
test_dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name='tests')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('zone', zone_dataset)
dataset_pool._add_dataset('tests', test_dataset)
values = zone_dataset.compute_variables(
['zone.aggregate(opus_core.tests.a_test_variable)'],
dataset_pool=dataset_pool)
should_be = array([45, 90])
self.assert_(ma.allclose(values, should_be, rtol=1e-6),
"Error in test_aggregate_fully_qualified_variable")
def test_alias_attribute(self):
# this tests an expression consisting of an alias for a primary attribute
expr = "p = persons"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"persons": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result = dataset.compute_variables([expr])
self.assertEqual(ma.allclose(result, [1, 5, 10], rtol=1e-7),
True,
msg="error in test_alias_attribute")
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
self.assertEqual(name.get_package_name(),
None,
msg="bad value for package")
self.assertEqual(name.get_dataset_name(),
None,
msg="bad value for dataset")
self.assert_(name.get_short_name().startswith('autogen'),
msg="bad value for shortname")
self.assertEqual(name.get_alias(), 'p', msg="bad value for alias")
self.assertNotEqual(name.get_autogen_class(),
None,
msg="bad value for autogen_class")
def test_casts_attribute(self):
expr1 = "persons.astype(float64)"
expr2 = "persons.astype(float64)**2"
expr3 = "(2*persons).astype(float64)"
error_msg = "Error in test_casts_attribute"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='tests',
table_data={
"persons": array([1, 5, 10]),
"id": array([1, 3, 4])
})
dataset = Dataset(in_storage=storage,
in_table_name='tests',
id_name="id",
dataset_name="tests")
result1 = dataset.compute_variables([expr1])
self.assertEqual(type(result1[0]), float64, error_msg)
self.assert_(ma.allclose(result1, array([1, 5, 10]), rtol=1e-6),
error_msg)
result2 = dataset.compute_variables([expr2])
self.assertEqual(type(result2[0]), float64, error_msg)
self.assert_(ma.allclose(result2, array([1, 25, 100]), rtol=1e-6),
error_msg)
result3 = dataset.compute_variables([expr3])
self.assertEqual(type(result3[0]), float64, error_msg)
self.assert_(ma.allclose(result3, array([2, 10, 20]), rtol=1e-6),
error_msg)
def setUp(self):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='households',
table_data={
'household_id': arange(10) + 1,
'grid_id': arange(-1, 9, 1) + 1,
'lucky': array([1, 0, 1, 0, 1, 1, 1, 1, 0, 0])
})
storage.write_table(
table_name='gridcells',
table_data={
'grid_id':
arange(15) + 1,
'filter':
array([0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1]),
'weight':
array([0.1, 9, 15, 2, 5, 1, 6, 2.1, .3, 4, 3, 1, 10, 8, 7])
})
#create households
self.households = Dataset(in_storage=storage,
in_table_name='households',
id_name="household_id",
dataset_name="household")
# create gridcells
self.gridcells = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="grid_id",
dataset_name="gridcell")
def test_join_datasets_with_2_ids(self):
from numpy import ma
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(
table_name='data1',
table_data={
'id1':array([2,4,2]),
'id2':array([1,2,3]),
'attr1':array([4,7,1]),
'attr2':array([100,0,1000]),
}
)
storage.write_table(
table_name='data2',
table_data={
'id1':array([4,2,2]),
'id2':array([2,3,1]),
'attr1':array([50,60,70])
}
)
ds1 = Dataset(in_storage=storage, in_table_name='data1', id_name=['id1', 'id2'], dataset_name='data1')
ds2 = Dataset(in_storage=storage, in_table_name='data2', id_name=['id1', 'id2'], dataset_name='data2')
ds1.join(ds2, 'attr1')
self.assertEqual(ma.allequal(ds1.get_attribute('attr1'), array([70,50,60])), True)
self.assertEqual(ma.allequal(ds1.get_attribute('attr2'), array([100,0,1000])), True)
def test_aggregate_bad_function(self):
# the 'function' argument must be a single name -- test this
expr = "zone.aggregate(2*gridcell.my_variable, function=3+4)"
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='zones',
table_data={
'zone_id': array([1, 2]),
})
storage.write_table(table_name='gridcells',
table_data={
'my_variable': array([4, 8, 0.5, 1]),
'grid_id': array([1, 2, 3, 4]),
'zone_id': array([1, 2, 1, 2]),
})
zone_dataset = Dataset(in_storage=storage,
in_table_name='zones',
id_name="zone_id",
dataset_name='zone')
gridcell_dataset = Dataset(in_storage=storage,
in_table_name='gridcells',
id_name="grid_id",
dataset_name='gridcell')
dataset_pool = DatasetPool()
dataset_pool._add_dataset('gridcell', gridcell_dataset)
dataset_pool._add_dataset('zone', zone_dataset)
self.assertRaises(ValueError,
zone_dataset.compute_variables, [expr],
dataset_pool=dataset_pool)
def test_unary_functions_fully_qualified_name(self):
# this tests expressions with unary functions applied to a fully qualified name
expr = "sqrt(opus_core.tests.a_test_variable)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])}
)
dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(
ma.allclose(result, should_be, rtol=1e-3), True, msg="error in test_unary_functions_fully_qualified_name"
)
# check that the access methods for the variable all return the correct values
name = VariableName(expr)
autogen = name.get_autogen_class()
self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable")
self.assertEqual(name.get_package_name(), None, msg="bad value for package")
self.assertEqual(name.get_dataset_name(), "tests", msg="bad value for dataset")
self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname")
self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias")
# make an instance of the class and check the dependencies (since the dependent variables
# all have fully-qualifed names we don't need to associate a dataset with the variable
# for this test)
self.assertEqual(
autogen().dependencies(), ["opus_core.tests.a_test_variable"], msg="dependencies are incorrect"
)
def __init__(self, variable_name, observed_data, filename=None, transformation=None, inverse_transformation=None,
filter=None, match=False, dependent_datasets={}, **kwargs):
""" 'variable_name' is a quantity about which we have data available.
'observed_data' is of type ObservedData, it is the grouping parent.
'filename' is the name of file where
the data is stored. It can be None, if the observed_data.directory is a cache.
'transformation' is an operation to be performed on the data (e.g. sqrt, log),
'inverse_transformation' is the inverse function of 'transformation'. If it not given, it
is determined automatically.
'filter' is a variable that will be applied to both, the observed data and the simulated data.
'match' (logical) determines if the dataset should be matched (by ids) with the simulated dataset. Elements
that don't match are eliminated from the simulated dataset.
'dependent_datasets' (if any) should be a dictionary of dataset_name:{'filename': filename, 'match': True|False, **kwargs}.
They will be added to the dataset_pool.
Remaining arguments are passed into DatasetFactory, thus it can contain information about how
to create the corresponding dataset.
"""
self.variable_name = VariableName(variable_name)
self.dataset_name = self.variable_name.get_dataset_name()
dataset_pool = observed_data.get_dataset_pool()
self.matching_datasets = {}
if dataset_pool is None:
kwargs.update({'in_storage':observed_data.get_storage(), 'in_table_name': filename})
try:
self.dataset = DatasetFactory().search_for_dataset(self.dataset_name, observed_data.get_package_order(), arguments=kwargs)
except: # take generic dataset
self.dataset = Dataset(dataset_name=self.dataset_name, **kwargs)
else:
self.dataset = dataset_pool.get_dataset(self.dataset_name)
if match:
self.add_match(self.dataset)
for dep_dataset_name, info in dependent_datasets.iteritems():
if dataset_pool is None:
dataset_pool = DatasetPool(storage=observed_data.get_storage(), package_order=observed_data.get_package_order())
info.update({'in_storage':observed_data.get_storage(), 'in_table_name': info.get('filename')})
del info['filename']
match = False
if 'match' in info.keys():
match = info['match']
del info['match']
try:
dep_dataset = DatasetFactory().search_for_dataset(dep_dataset_name, observed_data.get_package_order(), arguments=info)
except:
dep_dataset = Dataset(dataset_name=dep_dataset_name, **info)
dataset_pool.replace_dataset(dep_dataset_name, dep_dataset)
if match:
self.add_match(dep_dataset)
if self.variable_name.get_alias() not in self.dataset.get_known_attribute_names():
self.dataset.compute_variables([self.variable_name], dataset_pool=dataset_pool)
if filter is not None:
filter_values = self.dataset.compute_variables([filter], dataset_pool=dataset_pool)
idx = where(filter_values > 0)[0]
self.add_match(self.dataset, idx)
self.dataset.subset_by_index(idx)
self.transformation = transformation
self.inverse_transformation = inverse_transformation
if (self.transformation is not None) and (self.inverse_transformation is None):
self.inverse_transformation = self.transformation_pairs[self.transformation]
def __init__(self, *args, **kwargs):
Dataset.__init__(
self,
dataset_name="test_location",
id_name="id",
in_table_name=kwargs.get("in_table_name", "test_locations"),
in_storage=kwargs["in_storage"],
)
def __init__(self, *args, **kwargs):
Dataset.__init__(
self,
dataset_name="test_agent",
id_name="id",
in_table_name=kwargs.get('in_table_name','test_agents'),
in_storage=kwargs['in_storage']
)
def test_rand(self):
# test numpy.random.rand (this exercises 0-argument functions)
expr = "numpy.random.rand()"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
self.assert_(result >= 0 and result < 1, "Error in test_rand")
def test_numpy_arange_constant2(self):
# same as test_numpy_arange_constant, except provide 2 arguments
expr = "numpy.arange(2,5)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([2, 3, 4])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant2")
def test_numpy_arange_constant(self):
# test another constant - syntactically this looks like a method call, so it exercises that part of the code
expr = "numpy.arange(5)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"id": array([1, 2])})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
result = dataset.compute_variables([expr])
should_be = array([0, 1, 2, 3, 4])
self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_numpy_arange_constant")
def prepare_for_estimate(self, agents_for_estimation_storage, agents_for_estimation_table, agent_set, **kwargs):
estimation_set = Dataset(in_storage = agents_for_estimation_storage,
in_table_name=agents_for_estimation_table,
id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name())
hhs_est = HouseholdDataset(in_storage=agents_for_estimation_storage, in_table_name='households_for_estimation')
self.dataset_pool.replace_dataset('household', hhs_est)
self.dataset_pool.replace_dataset(estimation_set.get_dataset_name(), estimation_set)
spec, index = ChoiceModel.prepare_for_estimate(self, estimation_set, **kwargs)
return (spec, index, estimation_set)
def __init__(self, *args, **kwargs):
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='alldata',
table_data={
self.id_name:array([1]),
}
)
Dataset.__init__(self, in_storage=storage, in_table_name='alldata', id_name=self.id_name, dataset_name="alldata")
def function_tester(self, func, values, should_be, optional_args=None):
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="dataset", table_data={"my_variable": array(values), "id": arange(len(values))})
dataset = Dataset(in_storage=storage, in_table_name="dataset", id_name="id", dataset_name="mydataset")
if optional_args is None:
expr = "%s(my_variable)" % func
else:
expr = "%s(my_variable, %s)" % (func, optional_args)
result = dataset.compute_variables([expr])
self.assert_(ma.allclose(result, array(should_be), rtol=1e-6), "Error in " + func)
def create_edges(self, input_file_dir, input_file_name, output_file_name):
storage = StorageFactory().get_storage(type='tab_storage', subdir='store',
storage_location=input_file_dir)
dataset = Dataset(in_storage = storage, id_name = ['stop_id','sch_time'], in_table_name = input_file_name)
n = dataset.size()
trip_ids = dataset.get_attribute("stop_id")
unique_trip_ids = unique(trip_ids)
source_list = list()
target_list = list()
time_list = list()
for trip in unique_trip_ids:
idx = where(dataset.get_attribute("stop_id") == trip)[0]
nodes = dataset.get_attribute_by_index("node_id", idx)
times = dataset.get_attribute_by_index("sch_time", idx)
for inode in range(nodes.size-1):
source_list.append(nodes[inode])
target_list.append(nodes[inode+1])
time_list.append(times[inode+1] - times[inode])
storage = StorageFactory().get_storage('dict_storage')
storage.write_table(table_name='edges',
table_data={
'edge_id': arange(len(source_list))+1,
'source': array(source_list), #type=int64), # <<<< OUTPUT FIELD, USE array
'target': array(target_list), #type=int64), # <<<< OUTPUT FIELD, USE array
'cost': array(time_list, dtype=int32)
}
)
edges = Dataset(in_storage=storage, in_table_name='edges', id_name = "edge_id")
edges.write_dataset(attributes = ["source", "target", "cost"], out_storage = storage, out_table_name = output_file_name)
def test_where(self):
# test using the numpy where function
expr = "where(opus_core.test_agent.income>4, 100, 200)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="test_agents", table_data={"income": array([1, 5, 10, 3]), "id": array([1, 3, 4, 10])}
)
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([200, 100, 100, 200])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_where")
def test_true_false(self):
# make sure True and False can be used in an expression
expr = "array([True, False, False])"
# we're not actually using this dataset in the expression, but expressions are computed
# with respect to a dataset ...
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(table_name="test_agents", table_data={"income": array([10]), "id": array([1])})
dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent")
result = dataset.compute_variables([expr])
should_be = array([True, False, False])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-7), True, msg="Error in test_true_false")
def skip_test_dataset_qualified_name(self):
# this tests expressions with a dataset-qualified name
expr = "sqrt(tests.a_test_variable)"
storage = StorageFactory().get_storage("dict_storage")
storage.write_table(
table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])}
)
dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests")
result = dataset.compute_variables([expr])
should_be = array([3.16227766, 7.0710678, 10])
self.assertEqual(ma.allclose(result, should_be, rtol=1e-5), True)
def prepare_for_estimate_hh(self, estimation_storage, agents_for_estimation_table, agent_set,
persons_for_estimation_table=None, **kwargs):
estimation_set = Dataset(in_storage = estimation_storage,
in_table_name=agents_for_estimation_table,
id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name())
if persons_for_estimation_table is not None:
pers = PersonDataset(in_storage=estimation_storage, in_table_name=persons_for_estimation_table)
self.dataset_pool.replace_dataset('person', pers)
self.dataset_pool.replace_dataset(estimation_set.get_dataset_name(), estimation_set)
spec, index = ChoiceModel.prepare_for_estimate(self, estimation_set, **kwargs)
return (spec, index, estimation_set)
def prepare_for_estimate(self, estimation_storage, agents_for_estimation_table, agent_set,
households_for_estimation_table=None, **kwargs):
estimation_set = Dataset(in_storage = estimation_storage,
in_table_name=agents_for_estimation_table,
id_name=agent_set.get_id_name(), dataset_name=agent_set.get_dataset_name())
if households_for_estimation_table is not None:
hhs = HouseholdDataset(in_storage=estimation_storage, in_table_name='households_for_estimation')
self.dataset_pool.replace_dataset('household', hhs)
self.dataset_pool.replace_dataset(estimation_set.get_dataset_name(), estimation_set)
spec, index = HierarchicalChoiceModel.prepare_for_estimate(self, estimation_set, **kwargs)
return (spec, index, estimation_set)
