def check_parse_errors(self, variables): # check the variables in the expression library as indexed by the list 'variables'. errors = [] for (var_name, dataset_name, use, source, expr) in variables: # special case -- the 'constant' expression always passes if expr.strip()=='constant' and var_name=='constant': continue try: n = VariableName(expr) # check that the expression is of the correct form given the source if source=='primary attribute': if n.get_autogen_class() is not None: errors.append("Error - this is parsing as an expression rather than as a primary attribute: (%s, %s): %s" % (var_name, dataset_name, expr)) elif n.get_dataset_name() is None: errors.append("Error in primary attribute - missing dataset name: (%s, %s): %s" % (var_name, dataset_name, expr)) elif dataset_name!=n.get_dataset_name(): errors.append("Error in primary attribute - dataset name mismatch: (%s, %s): %s" % (var_name, dataset_name, expr)) elif n.get_package_name() is not None: errors.append("Error in primary attribute - shouldn't have package name: (%s, %s): %s" % (var_name, dataset_name, expr)) elif source=='expression': if n.get_autogen_class() is None: errors.append("Error - this doesn't seem to be an expression. Maybe it should be a Python class or primary attribute?: (%s, %s): %s" % (var_name, dataset_name, expr)) elif source=='Python class': if n.get_autogen_class() is not None: errors.append("Error - this is parsing as an expression rather than as a Python class reference: (%s, %s): %s" % (var_name, dataset_name, expr)) elif n.get_package_name() is None: errors.append("Error - missing package name in Python class reference: (%s, %s): %s" % (var_name, dataset_name, expr)) elif n.get_dataset_name() is None: errors.append("Error - missing dataset name in Python class reference: (%s, %s): %s" % (var_name, dataset_name, expr)) elif dataset_name!=n.get_dataset_name(): errors.append("Error - dataset name mismatch in Python class reference: (%s, %s): %s" % (var_name, dataset_name, expr)) else: errors.append("Unknown source type %s: (%s, %s): %s" % (source, var_name, dataset_name, expr)) except (SyntaxError, ValueError), e: errors.append("Parsing error: (%s, %s): %s" % (var_name, dataset_name, str(e)))
def test_alias_fully_qualified_variable(self): expr = "x = opus_core.tests.a_test_variable" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10, 50, 100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_alias_fully_qualified_variable") # check that the new var has x as an alias v = VariableName(expr) self.assertEqual(v.get_package_name(), None, msg="bad value for package_name") self.assertEqual(v.get_dataset_name(), 'tests', msg="bad value for dataset_name") self.assert_(v.get_short_name().startswith('autogen'), msg="bad value for shortname") self.assertEqual(v.get_alias(), 'x', msg="bad value for alias") self.assertNotEqual(v.get_autogen_class(), None, msg="bad value for autogen_class") # check that the alias has the correct value result2 = dataset.compute_variables(['x']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable")
def test_fully_qualified_variable(self): # this tests an expression consisting of a fully-qualified variable expr = "opus_core.test_agent.income_times_2" storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="test_agents", table_data={"income": array([1, 5, 10]), "id": array([1, 3, 4])}) dataset = Dataset(in_storage=storage, in_table_name="test_agents", id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array([2, 10, 20]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_variable") # check that expr is in the cache of known expressions # (normally we shouldn't be accessing this private field, but just this once ...) cache = VariableName._cache self.assert_(expr in cache, msg="did not find expr in cache") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), "opus_core", msg="bad value for package") self.assertEqual(name.get_dataset_name(), "test_agent", msg="bad value for dataset") self.assertEqual(name.get_short_name(), "income_times_2", msg="bad value for shortname") self.assertEqual(name.get_alias(), "income_times_2", msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(["income_times_2"]) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable") # check that the cache uses the variable name with whitespace removed oldsize = len(cache) expr_with_spaces = "opus_core . test_agent. income_times_2 " name2 = VariableName(expr_with_spaces) newsize = len(cache) self.assertEqual(oldsize, newsize, msg="caching error") self.assert_(expr_with_spaces not in cache, msg="caching error") self.assertEqual(expr_with_spaces, name2.get_expression(), msg="caching error") self.assertEqual(name2.get_short_name(), "income_times_2", msg="bad value for shortname")
def test_alias_fully_qualified_variable_same_name(self): expr = "a_test_variable = opus_core.tests.a_test_variable" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10, 50, 100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_alias_fully_qualified_variable") result2 = dataset.compute_variables(['a_test_variable']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable") v = VariableName(expr) # check that no autogen class was generated self.assertEqual(v.get_autogen_class(), None, msg="bad value for autogen_class") # check that the alias is correct self.assertEqual(v.get_alias(), 'a_test_variable', msg="bad value for alias")
def test_unary_functions_fully_qualified_name(self): # this tests expressions with unary functions applied to a fully qualified name expr = "sqrt(opus_core.tests.a_test_variable)" storage = StorageFactory().get_storage("dict_storage") storage.write_table( table_name="tests", table_data={"a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4])} ) dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([3.16227766, 7.0710678, 10]) self.assertEqual( ma.allclose(result, should_be, rtol=1e-3), True, msg="error in test_unary_functions_fully_qualified_name" ) # check that the access methods for the variable all return the correct values name = VariableName(expr) autogen = name.get_autogen_class() self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable") self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), "tests", msg="bad value for dataset") self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname") self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias") # make an instance of the class and check the dependencies (since the dependent variables # all have fully-qualifed names we don't need to associate a dataset with the variable # for this test) self.assertEqual( autogen().dependencies(), ["opus_core.tests.a_test_variable"], msg="dependencies are incorrect" )
def test_alias_attribute_same_name(self): # this tests an expression consisting of an alias for a primary attribute that is the same name as the primary attribute expr = "persons = persons" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "persons": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) self.assertEqual(ma.allclose(result, [1, 5, 10], rtol=1e-7), True, msg="error in test_alias_attribute") name = VariableName(expr) self.assertEqual(name.get_short_name(), 'persons', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'persons', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
def test_constants(self): # test an expression involving two dataset names, one of which is *_constant expr = "test_agent.age<=opus_constant.young_age" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ "age": array([30, 20, 60, 80]), "id": array([1, 3, 4, 10]) }) storage.write_table(table_name='opus_constants', table_data={ "young_age": array([35]), "opus_constant_id": array([1]) }) dataset_pool = DatasetPool(storage=storage) # Test that the dataset name is correct for expr. It should be test_agent -- opus_constant just holds constants, # and is ignored as far as finding the dataset name for the expression. name = VariableName(expr) autogen = name.get_autogen_class() self.assertEqual(name.get_package_name(), None) self.assertEqual(name.get_dataset_name(), 'test_agent') # make an instance of the class and check the dependencies (it shouldn't depend on urbansim_constant) self.assertEqual(autogen().dependencies(), ['test_agent.age']) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr], dataset_pool=dataset_pool) should_be = array([True, True, False, False]) self.assertEqual(ma.allequal(result, should_be), True)
def test_fully_qualified_DDD_SSS_variable(self): # this should use the test variable a_test_SSS_variable_DDD_SSS expr = "opus_core.tests.a_test_squid_variable_42_clam" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10,50,100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_DDD_SSS_variable") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset") self.assertEqual(name.get_short_name(), 'a_test_squid_variable_42_clam', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'a_test_squid_variable_42_clam', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(['a_test_squid_variable_42_clam']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_squid_variable_42_clam")
def test_unary_functions_fully_qualified_name(self): # this tests expressions with unary functions applied to a fully qualified name expr = "sqrt(opus_core.tests.a_test_variable)" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([3.16227766, 7.0710678, 10]) self.assertEqual(ma.allclose(result, should_be, rtol=1e-3), True, msg="error in test_unary_functions_fully_qualified_name") # check that the access methods for the variable all return the correct values name = VariableName(expr) autogen = name.get_autogen_class() self.assert_(issubclass(autogen, Variable), msg="autogen'd class isn't a Variable") self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset") self.assertEqual(name.get_short_name(), autogen.__name__, msg="bad value for shortname") self.assertEqual(name.get_alias(), autogen.__name__, msg="bad value for alias") # make an instance of the class and check the dependencies (since the dependent variables # all have fully-qualifed names we don't need to associate a dataset with the variable # for this test) self.assertEqual(autogen().dependencies(), ['opus_core.tests.a_test_variable'], msg="dependencies are incorrect")
def test_alias_attribute(self): # this tests an expression consisting of an alias for a primary attribute expr = "p = persons" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "persons": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) self.assertEqual(ma.allclose(result, [1, 5, 10], rtol=1e-7), True, msg="error in test_alias_attribute") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), None, msg="bad value for dataset") self.assert_(name.get_short_name().startswith('autogen'), msg="bad value for shortname") self.assertEqual(name.get_alias(), 'p', msg="bad value for alias") self.assertNotEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
def test_constants(self): # test an expression involving two dataset names, one of which is *_constant expr = "test_agent.age<=opus_constant.young_age" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='test_agents', table_data={ "age":array([30,20,60,80]), "id":array([1,3,4,10]) } ) storage.write_table( table_name='opus_constants', table_data={ "young_age":array([35]), "opus_constant_id":array([1]) } ) dataset_pool = DatasetPool(storage=storage) # Test that the dataset name is correct for expr. It should be test_agent -- opus_constant just holds constants, # and is ignored as far as finding the dataset name for the expression. name = VariableName(expr) autogen = name.get_autogen_class() self.assertEqual(name.get_package_name(), None) self.assertEqual(name.get_dataset_name(), 'test_agent') # make an instance of the class and check the dependencies (it shouldn't depend on urbansim_constant) self.assertEqual(autogen().dependencies(), ['test_agent.age']) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr], dataset_pool=dataset_pool) should_be = array( [True,True,False,False] ) self.assertEqual( ma.allequal( result, should_be), True)
def test_fully_qualified_variable(self): # this tests an expression consisting of a fully-qualified variable expr = "opus_core.test_agent.income_times_2" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='test_agents', table_data={ "income": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='test_agents', id_name="id", dataset_name="test_agent") result = dataset.compute_variables([expr]) should_be = array([2, 10, 20]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_variable") # check that expr is in the cache of known expressions # (normally we shouldn't be accessing this private field, but just this once ...) cache = VariableName._cache self.assert_(expr in cache, msg="did not find expr in cache") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'test_agent', msg="bad value for dataset") self.assertEqual(name.get_short_name(), 'income_times_2', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'income_times_2', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(['income_times_2']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable") # check that the cache uses the variable name with whitespace removed oldsize = len(cache) expr_with_spaces = "opus_core . test_agent. income_times_2 " name2 = VariableName(expr_with_spaces) newsize = len(cache) self.assertEqual(oldsize, newsize, msg="caching error") self.assert_(expr_with_spaces not in cache, msg="caching error") self.assertEqual(expr_with_spaces, name2.get_expression(), msg="caching error") self.assertEqual(name2.get_short_name(), 'income_times_2', msg="bad value for shortname")
def test_dataset_qualified_attribute(self): expr = "tests.persons" storage = StorageFactory().get_storage("dict_storage") storage.write_table(table_name="tests", table_data={"persons": array([1, 5, 10]), "id": array([1, 3, 4])}) dataset = Dataset(in_storage=storage, in_table_name="tests", id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) self.assertEqual(ma.allclose(result, [1, 5, 10], rtol=1e-7), True, msg="error in test_attribute") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), "tests", msg="bad value for dataset") self.assertEqual(name.get_short_name(), "persons", msg="bad value for shortname") self.assertEqual(name.get_alias(), "persons", msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
def test_alias_attribute_same_name(self): # this tests an expression consisting of an alias for a primary attribute that is the same name as the primary attribute expr = "persons = persons" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "persons":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) self.assertEqual(ma.allclose(result, [1,5,10], rtol=1e-7), True, msg="error in test_alias_attribute") name = VariableName(expr) self.assertEqual(name.get_short_name(), 'persons', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'persons', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
def add_prefix_to_variable_names(self, variable_names, dataset, variable_package, resources): """Add a prefix of 'package.dataset_name.' to variable_names from resources. """ if resources is None: return if not isinstance(variable_names, list): variable_names = [variable_names] for variable_name in variable_names: variable_string = resources.get(variable_name, None) if variable_string is not None: variable_string_name = VariableName(variable_string) if (variable_string_name.get_dataset_name() == None) and \ (variable_string_name.get_autogen_class() is None) : add_string = "" if variable_string_name.get_package_name() == None: add_string = "%s." % variable_package add_string = add_string + dataset.get_dataset_name() + "." resources.merge({ variable_name:add_string+variable_string})
def add_prefix_to_variable_names(self, variable_names, dataset, variable_package, resources): """Add a prefix of 'package.dataset_name.' to variable_names from resources. """ if resources is None: return if not isinstance(variable_names, list): variable_names = [variable_names] for variable_name in variable_names: variable_string = resources.get(variable_name, None) if variable_string is not None: variable_string_name = VariableName(variable_string) if (variable_string_name.get_dataset_name() == None) and \ (variable_string_name.get_autogen_class() is None) : add_string = "" if variable_string_name.get_package_name() == None: add_string = "%s." % variable_package add_string = add_string + dataset.get_dataset_name() + "." resources.merge( {variable_name: add_string + variable_string})
def test_alias_attribute(self): # this tests an expression consisting of an alias for a primary attribute expr = "p = persons" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "persons":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) self.assertEqual(ma.allclose(result, [1,5,10], rtol=1e-7), True, msg="error in test_alias_attribute") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), None, msg="bad value for package") self.assertEqual(name.get_dataset_name(), None, msg="bad value for dataset") self.assert_(name.get_short_name().startswith('autogen'), msg="bad value for shortname") self.assertEqual(name.get_alias(), 'p', msg="bad value for alias") self.assertNotEqual(name.get_autogen_class(), None, msg="bad value for autogen_class")
def test_alias_fully_qualified_variable_same_name(self): expr = "a_test_variable = opus_core.tests.a_test_variable" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10,50,100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_alias_fully_qualified_variable") result2 = dataset.compute_variables(['a_test_variable']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable") v = VariableName(expr) # check that no autogen class was generated self.assertEqual(v.get_autogen_class(), None, msg="bad value for autogen_class") # check that the alias is correct self.assertEqual(v.get_alias(), 'a_test_variable', msg="bad value for alias")
def test_fully_qualified_DDD_SSS_variable(self): # this should use the test variable a_test_SSS_variable_DDD_SSS expr = "opus_core.tests.a_test_squid_variable_42_clam" storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='tests', table_data={ "a_dependent_variable": array([1, 5, 10]), "id": array([1, 3, 4]) }) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10, 50, 100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_fully_qualified_DDD_SSS_variable") # check that the access methods for the variable all return the correct values name = VariableName(expr) self.assertEqual(name.get_package_name(), 'opus_core', msg="bad value for package") self.assertEqual(name.get_dataset_name(), 'tests', msg="bad value for dataset") self.assertEqual(name.get_short_name(), 'a_test_squid_variable_42_clam', msg="bad value for shortname") self.assertEqual(name.get_alias(), 'a_test_squid_variable_42_clam', msg="bad value for alias") self.assertEqual(name.get_autogen_class(), None, msg="bad value for autogen_class") # test that the variable can now also be accessed using its short name in an expression result2 = dataset.compute_variables(['a_test_squid_variable_42_clam']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_squid_variable_42_clam")
def test_alias_fully_qualified_variable(self): expr = "x = opus_core.tests.a_test_variable" storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='tests', table_data={ "a_dependent_variable":array([1,5,10]), "id":array([1,3,4]) } ) dataset = Dataset(in_storage=storage, in_table_name='tests', id_name="id", dataset_name="tests") result = dataset.compute_variables([expr]) should_be = array([10,50,100]) self.assert_(ma.allclose(result, should_be, rtol=1e-6), "Error in test_alias_fully_qualified_variable") # check that the new var has x as an alias v = VariableName(expr) self.assertEqual(v.get_package_name(), None, msg="bad value for package_name") self.assertEqual(v.get_dataset_name(), 'tests', msg="bad value for dataset_name") self.assert_(v.get_short_name().startswith('autogen'), msg="bad value for shortname") self.assertEqual(v.get_alias(), 'x', msg="bad value for alias") self.assertNotEqual(v.get_autogen_class(), None, msg="bad value for autogen_class") # check that the alias has the correct value result2 = dataset.compute_variables(['x']) self.assert_(ma.allclose(result2, should_be, rtol=1e-6), "Error in accessing a_test_variable")