def test_settings(self):
settings = {'superset_right_in_is_subset': False,
'set_right_in_contains': None}
grammar = Grammar(settings)
# Checking the new settings:
eq_(grammar.get_setting("superset_right_in_is_subset"), False)
eq_(grammar.get_setting("set_right_in_contains"), None)
# Everything else must have not changed:
eq_(grammar.get_setting("optional_positive_sign"), True)
def test_generators(self):
mock_generator = lambda: None
generators = {'string': mock_generator}
grammar = Grammar(None, generators)
# Checking the new generators:
eq_(grammar.get_custom_generator("string"), mock_generator)
# Everything else must have not changed:
eq_(grammar.get_custom_generator("operation"), None)
eq_(grammar.get_custom_generator("number"), None)
def test_generators(self):
mock_generator = lambda: None
generators = {'string': mock_generator}
grammar = Grammar(None, generators)
# Checking the new generators:
eq_(grammar.get_custom_generator("string"), mock_generator)
# Everything else must have not changed:
eq_(grammar.get_custom_generator("operation"), None)
eq_(grammar.get_custom_generator("number"), None)
def test_adding_grammar(self):
"""It should be possible to add grammars after instantiation."""
castilian_grammar = Grammar(decimal_separator=",",
thousands_separator=".")
mgr = EvaluableParseManager(self.symbol_table, Grammar())
mgr.add_parser("es", castilian_grammar)
parse_tree = mgr.parse(u'tráfico:peatones_cruzando_calle <= 3,00',
"es")
expected_tree = EvaluableParseTree(
LessEqual(PedestriansCrossingRoad(), Number(3.0)))
eq_(parse_tree, expected_tree)
def test_parsing_with_localized_grammars(self):
castilian_grammar = Grammar(decimal_separator=",",
thousands_separator=".")
mgr = ConvertibleParseManager(Grammar(), es=castilian_grammar)
parse_tree = mgr.parse(u"tráfico:peatones_cruzando_calle <= 3,00",
"es")
expected_tree = ConvertibleParseTree(
LessEqual(
PlaceholderVariable("peatones_cruzando_calle", (u"tráfico", )),
Number(3.0)))
eq_(parse_tree, expected_tree)
def test_parsing_with_localized_grammars(self):
castilian_grammar = Grammar(decimal_separator=",",
thousands_separator=".")
mgr = EvaluableParseManager(self.symbol_table,
Grammar(),
es=castilian_grammar)
parse_tree = mgr.parse(u"tráfico:peatones_cruzando_calle <= 3,00",
"es")
expected_tree = EvaluableParseTree(
LessEqual(PedestriansCrossingRoad(), Number(3.0)))
eq_(parse_tree, expected_tree)
def test_settings(self):
settings = {
'superset_right_in_is_subset': False,
'set_right_in_contains': None
}
grammar = Grammar(settings)
# Checking the new settings:
eq_(grammar.get_setting("superset_right_in_is_subset"), False)
eq_(grammar.get_setting("set_right_in_contains"), None)
# Everything else must have not changed:
eq_(grammar.get_setting("optional_positive_sign"), True)
def test_parsing_with_no_localized_grammars(self):
mgr = ConvertibleParseManager(Grammar())
parse_tree1 = mgr.parse('message == "2009-07-13"')
parse_tree2 = mgr.parse('message == "2009-07-13"', None)
expected_tree = ConvertibleParseTree(
Equal(PlaceholderVariable("message"), String("2009-07-13")))
eq_(parse_tree1, parse_tree2)
eq_(parse_tree1, expected_tree)
def test_parsing_with_no_localized_grammars(self):
mgr = EvaluableParseManager(self.symbol_table, Grammar())
parse_tree1 = mgr.parse('message == "2009-07-13"')
parse_tree2 = mgr.parse('message == "2009-07-13"', None)
expected_tree = EvaluableParseTree(
Equal(String("Hello world"), String("2009-07-13")))
eq_(parse_tree1, parse_tree2)
eq_(parse_tree1, expected_tree)
def test_parsing_with_defined_grammar_but_no_available_translations(self):
"""
When an expression is written in an supported grammar but there are no
translated bindings, the default names must be used along with the
custom grammar.
"""
french_grammar = Grammar(decimal_separator=",",
thousands_separator=".")
mgr = EvaluableParseManager(self.symbol_table,
Grammar(),
fr=french_grammar)
# French grammar is not supported:
parse_tree = mgr.parse("traffic:pedestrians_crossing_road <= 3,0",
"fr")
expected_tree = EvaluableParseTree(
LessEqual(PedestriansCrossingRoad(), Number(3)))
eq_(parse_tree, expected_tree)
def test_evaluating_expressions(self):
"""Managers should be able to evaluate the expressions too."""
mgr = EvaluableParseManager(self.symbol_table, Grammar())
context = {'pedestrians_crossroad': (u"gustavo", u"carla")}
evaluation1 = mgr.evaluate("traffic:pedestrians_crossing_road <= 3",
None, context)
evaluation2 = mgr.evaluate(
u'"carla" ∈ traffic:pedestrians_crossing_road', None, context)
evaluation3 = mgr.evaluate("traffic:pedestrians_crossing_road > 2",
None, context)
ok_(evaluation1)
ok_(evaluation2)
assert_false(evaluation3)
class TestEvaluableParser(object):
"""Tests for the evaluable parser."""
global_objects = {
'bool': BoolVar(),
'message': String("Hello world"),
'foo': PermissiveFunction,
}
traffic_objects = {
'traffic_light': TrafficLightVar(),
'pedestrians_crossing_road': PedestriansCrossingRoad(),
'drivers_awaiting_green_light': DriversAwaitingGreenLightVar(),
'traffic_violation': TrafficViolationFunc,
}
root_namespace = Namespace(global_objects, {
'traffic': Namespace(traffic_objects),
})
parser = EvaluableParser(Grammar(), root_namespace)
#{ Tests for the parse action that makes the variables
def test_existing_variable_without_namespace(self):
parse_tree = self.parser("~ bool")
eq_(parse_tree.root_node, Not(self.global_objects['bool']))
def test_existing_variable_with_namespace(self):
parse_tree = self.parser('traffic:traffic_light == "green"')
expected_node = Equal(self.traffic_objects['traffic_light'],
String("green"))
eq_(parse_tree.root_node, expected_node)
def test_non_existing_variable_without_namespace(self):
assert_raises(ScopeError, self.parser, "~ non_existing_var")
def test_non_existing_variable_with_namespace(self):
assert_raises(ScopeError, self.parser, "~ traffic:non_existing_var")
def test_variable_in_non_existing_namespace(self):
assert_raises(ScopeError, self.parser, "~ bar:foo")
def test_function_instead_of_variable(self):
# "foo" is a function, so it cannot be used as a variable (without
# parenthesis):
try:
self.parser('~ foo')
except BadExpressionError, exc:
eq_('"foo" represents a function, not a variable', unicode(exc))
else:
def create_grammar(self):
root_table = SymbolTable(
"root",
map(lambda f: Bind(f['name'], GeometryProperty(f['name'])),
self.fields))
tokens = {
'not': "not",
'eq': "==",
'ne': "!=",
'belongs_to': "in",
'is_subset': "are included in"
}
grammar = Grammar(**tokens)
self.parse_manager = EvaluableParseManager(root_table, grammar)
def create_grammar(self):
root_table = SymbolTable("Shop_root",
map( lambda f: Bind(f['name'], GeometryProperty(f['name'])), self.fields )
)
tokens = {
'not': 'not',
'eq': '==',
'ne': '!=',
'belongs_to': 'in',
'is_subset': 'are included in',
'or': "or",
'and': 'and'
}
grammar = Grammar(**tokens)
self.parse_manager = EvaluableParseManager(root_table, grammar)
def test_unlimited_caching(self):
"""
The cache may have no limit.
If so, the cache counter will be disabled.
"""
manager = ConvertibleParseManager(Grammar(), cache_limit=None)
manager.parse('today == "2009-07-13"')
manager.parse('yesterday < "2009-07-13"')
manager.parse('tomorrow > "2009-07-13"')
manager.parse('hoy > "1999-01-06"')
manager.parse('ayer > "1999-01-06"')
# Checking the cache:
eq_(manager._cache.counter, 5)
eq_(len(manager._cache.cache_by_locale[None]), 5)
eq_(len(manager._cache.latest_expressions), 5)
def test_parsing_with_undefined_grammar_but_available_translations(self):
"""
When an expression is written in an unsupported grammar, a parser
based on the generic grammar must be created and used.
The respective translated bindings must be used if available.
"""
log_handler = LoggingHandlerFixture()
mgr = EvaluableParseManager(self.symbol_table, Grammar())
# Castilian grammar is not supported:
parse_tree = mgr.parse(u"tráfico:peatones_cruzando_calle <= 3.0", "es")
expected_tree = EvaluableParseTree(
LessEqual(PedestriansCrossingRoad(), Number(3)))
eq_(parse_tree, expected_tree)
# Checking the log:
info = "Generated parser for unknown grammar 'es'"
ok_(info in log_handler.handler.messages['info'])
log_handler.undo()
def test_parsing_with_undefined_grammar_and_no_translated_bindings(self):
"""
When an expression is written in an unsupported grammar, a parser
based on the generic grammar must be created and used.
If there are no translated bindings, the default names must be used.
"""
log_handler = LoggingHandlerFixture()
mgr = EvaluableParseManager(self.symbol_table, Grammar())
# French grammar is not supported:
parse_tree = mgr.parse("traffic:pedestrians_crossing_road <= 3.0",
"fr")
expected_tree = EvaluableParseTree(
LessEqual(PedestriansCrossingRoad(), Number(3)))
eq_(parse_tree, expected_tree)
# Checking the log:
info = "Generated parser for unknown grammar %s" % repr('fr')
ok_(info in log_handler.handler.messages['info'])
log_handler.undo()
def test_abstract_methods(self):
parser = Parser(Grammar())
assert_raises(NotImplementedError, parser.make_variable, None)
assert_raises(NotImplementedError, parser.make_function, None)
def test_custom_tokens(self):
"""The tokens can be customized using the keyword arguments."""
grammar = Grammar(eq="=", ne="<>")
# Checking the two new tokens:
eq_(grammar.get_token("eq"), "=")
eq_(grammar.get_token("ne"), "<>")
# Everything else must have not changed:
eq_(grammar.get_token("not"), "~")
eq_(grammar.get_token("and"), "&")
eq_(grammar.get_token("or"), "|")
eq_(grammar.get_token("xor"), "^")
eq_(grammar.get_token("lt"), "<")
eq_(grammar.get_token("gt"), ">")
eq_(grammar.get_token("le"), "<=")
eq_(grammar.get_token("ge"), ">=")
eq_(grammar.get_token("belongs_to"), u"∈")
eq_(grammar.get_token("is_subset"), u"⊂")
eq_(grammar.get_token("set_start"), "{")
eq_(grammar.get_token("set_end"), "}")
eq_(grammar.get_token("element_separator"), ",")
eq_(grammar.get_token("string_start"), '"')
eq_(grammar.get_token("string_end"), '"')
eq_(grammar.get_token("group_start"), "(")
eq_(grammar.get_token("group_end"), ")")
eq_(grammar.get_token("arguments_start"), "(")
eq_(grammar.get_token("arguments_end"), ")")
eq_(grammar.get_token("arguments_separator"), ",")
eq_(grammar.get_token("positive_sign"), "+")
eq_(grammar.get_token("negative_sign"), "-")
eq_(grammar.get_token("decimal_separator"), ".")
eq_(grammar.get_token("thousands_separator"), ",")
eq_(grammar.get_token("identifier_spacing"), "_")
eq_(grammar.get_token("namespace_separator"), ":")
def test_get_all_tokens(self):
grammar = Grammar(eq="=", ne="<>")
expected = grammar.default_tokens.copy()
expected['eq'] = '='
expected['ne'] = '<>'
eq_(grammar.get_all_tokens(), expected)
def test_custom_tokens(self):
"""The tokens can be customized using the keyword arguments."""
grammar = Grammar(eq="=", ne="<>")
# Checking the two new tokens:
eq_(grammar.get_token("eq"), "=")
eq_(grammar.get_token("ne"), "<>")
# Everything else must have not changed:
eq_(grammar.get_token("not"), "~")
eq_(grammar.get_token("and"), "&")
eq_(grammar.get_token("or"), "|")
eq_(grammar.get_token("xor"), "^")
eq_(grammar.get_token("lt"), "<")
eq_(grammar.get_token("gt"), ">")
eq_(grammar.get_token("le"), "<=")
eq_(grammar.get_token("ge"), ">=")
eq_(grammar.get_token("belongs_to"), u"∈")
eq_(grammar.get_token("is_subset"), u"⊂")
eq_(grammar.get_token("set_start"), "{")
eq_(grammar.get_token("set_end"), "}")
eq_(grammar.get_token("element_separator"), ",")
eq_(grammar.get_token("string_start"), '"')
eq_(grammar.get_token("string_end"), '"')
eq_(grammar.get_token("group_start"), "(")
eq_(grammar.get_token("group_end"), ")")
eq_(grammar.get_token("arguments_start"), "(")
eq_(grammar.get_token("arguments_end"), ")")
eq_(grammar.get_token("arguments_separator"), ",")
eq_(grammar.get_token("positive_sign"), "+")
eq_(grammar.get_token("negative_sign"), "-")
eq_(grammar.get_token("decimal_separator"), ".")
eq_(grammar.get_token("thousands_separator"), ",")
eq_(grammar.get_token("identifier_spacing"), "_")
eq_(grammar.get_token("namespace_separator"), ":")
# 1: create a static Symbol Table
root_table = SymbolTable(
"root",
(
# variablet taken from context
Bind("age", NumberVariable('age')),
Bind("name", StringVariable('name')),
Bind("birthdate", DateVariable("birthdate")),
Bind("tags", SetVariable("tags")),
# constants
Bind("majority", Number(18)),
))
# 2: create the parser with se symbol table and the grammar (customized)
grammar = Grammar(belongs_to='in')
parse_manager = EvaluableParseManager(root_table, grammar)
# 3: compile a expression
# compiled_expression = parse_manager.parse('age < majority & "o" in name & birthdate > "1983-02-02"')
# check the command line args
if len(sys.argv) == 1:
print("try something like %s '\"a\" in name'" % sys.argv[0])
exit(1)
stmt = " ".join(sys.argv[1:])
print("searching with expression %r" % stmt)
try:
compiled_expression = parse_manager.parse(stmt)
except pyparsing.ParseException as e:
attrs = dict(expr=stmt,
class TestDefaultGrammar(object):
"""Tests for the grammar with the default properties, at least initially."""
def setUp(self):
self.grammar = Grammar()
# Token handling stuff
def test_default_tokens(self):
"""All the tokens must have an initial value."""
# Logical connectives:
eq_(self.grammar.get_token("not"), "~")
eq_(self.grammar.get_token("and"), "&")
eq_(self.grammar.get_token("or"), "|")
eq_(self.grammar.get_token("xor"), "^")
# Relational operators
eq_(self.grammar.get_token("eq"), "==")
eq_(self.grammar.get_token("ne"), "!=")
eq_(self.grammar.get_token("lt"), "<")
eq_(self.grammar.get_token("gt"), ">")
eq_(self.grammar.get_token("le"), "<=")
eq_(self.grammar.get_token("ge"), ">=")
# Set operators
eq_(self.grammar.get_token("belongs_to"), u"∈")
eq_(self.grammar.get_token("is_subset"), u"⊂")
eq_(self.grammar.get_token("set_start"), "{")
eq_(self.grammar.get_token("set_end"), "}")
eq_(self.grammar.get_token("element_separator"), ",")
# Grouping marks
eq_(self.grammar.get_token("string_start"), '"')
eq_(self.grammar.get_token("string_end"), '"')
eq_(self.grammar.get_token("group_start"), "(")
eq_(self.grammar.get_token("group_end"), ")")
# Function call stuff
eq_(self.grammar.get_token("arguments_start"), "(")
eq_(self.grammar.get_token("arguments_end"), ")")
eq_(self.grammar.get_token("arguments_separator"), ",")
# Numeric stuff
eq_(self.grammar.get_token("positive_sign"), "+")
eq_(self.grammar.get_token("negative_sign"), "-")
eq_(self.grammar.get_token("decimal_separator"), ".")
eq_(self.grammar.get_token("thousands_separator"), ",")
# Miscellaneous
eq_(self.grammar.get_token("identifier_spacing"), "_")
eq_(self.grammar.get_token("namespace_separator"), ":")
def test_get_all_tokens(self):
eq_(self.grammar.get_all_tokens(), self.grammar.default_tokens)
def test_setting_existing_token(self):
self.grammar.set_token("negative_sign", "!")
eq_(self.grammar.get_token("negative_sign"), "!")
def test_requesting_non_existing_token(self):
assert_raises(GrammarError, self.grammar.get_token, "non_existing")
def test_setting_non_existing_token(self):
assert_raises(GrammarError, self.grammar.set_token, "non_existing",
"-")
# Setting handling stuff
def test_default_settings(self):
eq_(self.grammar.get_setting("superset_right_in_is_subset"), True)
eq_(self.grammar.get_setting("set_right_in_contains"), True)
eq_(self.grammar.get_setting("optional_positive_sign"), True)
def test_setting_existing_setting(self):
self.grammar.set_setting("set_right_in_contains", False)
eq_(self.grammar.get_setting("set_right_in_contains"), False)
def test_requesting_non_existing_setting(self):
assert_raises(GrammarError, self.grammar.get_setting, "non_existing")
def test_setting_non_existing_setting(self):
assert_raises(GrammarError, self.grammar.set_setting, "non_existing",
None)
# Custom generator handling stuff
def test_no_custom_generators_by_default(self):
"""There must not be custom generators by default."""
eq_(self.grammar.get_custom_generator("operation"), None)
eq_(self.grammar.get_custom_generator("string"), None)
eq_(self.grammar.get_custom_generator("number"), None)
def test_setting_existing_generator(self):
mock_generator = lambda: None
self.grammar.set_custom_generator("number", mock_generator)
eq_(self.grammar.get_custom_generator("number"), mock_generator)
def test_requesting_non_existing_generator(self):
assert_raises(GrammarError, self.grammar.get_custom_generator,
"non_existing")
def test_setting_non_existing_generator(self):
mock_generator = lambda: None
assert_raises(GrammarError, self.grammar.set_custom_generator,
"non_existing", mock_generator)
def setUp(self):
self.grammar = Grammar()
def test_abstract_methods(self):
mgr = ParseManager(Grammar())
assert_raises(NotImplementedError, mgr._define_parser, None, "es")
__all__ = ("parse_manager")
# Defining the grammars:
english_tokens = {
'not': "not",
'and': "and",
'or': "or",
'eq': "is",
'ne': "is not",
'belongs_to': "is contained in",
'is_subset': "is sub-set of",
}
english_grammar = Grammar(**english_tokens)
dutch_tokens = {
'not': "niet",
'and': "en",
'or': "of",
'eq': "is",
'ne': "is geen",
'belongs_to': "is opgenomen in",
'is_subset': "is onderdeel van",
}
dutch_grammar = Grammar(**dutch_tokens)
castilian_tokens = {
'not': "no",
'and': "y",
def setUp(self):
castilian_grammar = Grammar(decimal_separator=",",
thousands_separator=".")
self.manager = ConvertibleParseManager(Grammar(),
cache_limit=3,
es=castilian_grammar)
SymbolTable("today", []),
SymbolTable("days_of_history", []),
SymbolTable("day_of_week", []),
SymbolTable("month", [])
)
)
from booleano.parser import Grammar
new_tokens = {
'and': '&&',
'or': '||',
'belongs_to': 'in',
'is_subset': 'subset of'
}
english_grammar = Grammar(**new_tokens)
from booleano.parser import EvaluableParseManager
parse_manager = EvaluableParseManager(root_table, english_grammar)
buy_parser = parse_manager.parse(buy_conditions)
sell_parser = parse_manager.parse(sell_conditions)
print(buy_parser)
print(sell_parser)
sp = StockProcessing(data_dir, date_fmt, (date_column, open_column, close_column))
sp.generate_sold_stocks()
sp.buy_stocks(buy_parser, sell_parser)
sa = StockAnalysis(sp.order_history, sp.owned_stocks, sp.dates_arr, sp.date_keyed, buy_conditions, sell_conditions)
class TestDefaultGrammar(object):
"""Tests for the grammar with the default properties, at least initially."""
def setUp(self):
self.grammar = Grammar()
#{ Token handling stuff
def test_default_tokens(self):
"""All the tokens must have an initial value."""
# Logical connectives:
eq_(self.grammar.get_token("not"), "~")
eq_(self.grammar.get_token("and"), "&")
eq_(self.grammar.get_token("or"), "|")
eq_(self.grammar.get_token("xor"), "^")
# Relational operators
eq_(self.grammar.get_token("eq"), "==")
eq_(self.grammar.get_token("ne"), "!=")
eq_(self.grammar.get_token("lt"), "<")
eq_(self.grammar.get_token("gt"), ">")
eq_(self.grammar.get_token("le"), "<=")
eq_(self.grammar.get_token("ge"), ">=")
# Set operators
eq_(self.grammar.get_token("belongs_to"), u"∈")
eq_(self.grammar.get_token("is_subset"), u"⊂")
eq_(self.grammar.get_token("set_start"), "{")
eq_(self.grammar.get_token("set_end"), "}")
eq_(self.grammar.get_token("element_separator"), ",")
# Grouping marks
eq_(self.grammar.get_token("string_start"), '"')
eq_(self.grammar.get_token("string_end"), '"')
eq_(self.grammar.get_token("group_start"), "(")
eq_(self.grammar.get_token("group_end"), ")")
# Function call stuff
eq_(self.grammar.get_token("arguments_start"), "(")
eq_(self.grammar.get_token("arguments_end"), ")")
eq_(self.grammar.get_token("arguments_separator"), ",")
# Numeric stuff
eq_(self.grammar.get_token("positive_sign"), "+")
eq_(self.grammar.get_token("negative_sign"), "-")
eq_(self.grammar.get_token("decimal_separator"), ".")
eq_(self.grammar.get_token("thousands_separator"), ",")
# Miscellaneous
eq_(self.grammar.get_token("identifier_spacing"), "_")
eq_(self.grammar.get_token("namespace_separator"), ":")
def test_setting_existing_token(self):
self.grammar.set_token("negative_sign", "!")
eq_(self.grammar.get_token("negative_sign"), "!")
def test_requesting_non_existing_token(self):
assert_raises(GrammarError, self.grammar.get_token, "non_existing")
def test_setting_non_existing_token(self):
assert_raises(GrammarError, self.grammar.set_token, "non_existing", "-")
#{ Setting handling stuff
def test_default_settings(self):
eq_(self.grammar.get_setting("superset_right_in_is_subset"), True)
eq_(self.grammar.get_setting("set_right_in_contains"), True)
eq_(self.grammar.get_setting("optional_positive_sign"), True)
def test_setting_existing_setting(self):
self.grammar.set_setting("set_right_in_contains", False)
eq_(self.grammar.get_setting("set_right_in_contains"), False)
def test_requesting_non_existing_setting(self):
assert_raises(GrammarError, self.grammar.get_setting, "non_existing")
def test_setting_non_existing_setting(self):
assert_raises(GrammarError, self.grammar.set_setting, "non_existing",
None)
#{ Custom generator handling stuff
def test_no_custom_generators_by_default(self):
"""There must not be custom generators by default."""
eq_(self.grammar.get_custom_generator("operation"), None)
eq_(self.grammar.get_custom_generator("string"), None)
eq_(self.grammar.get_custom_generator("number"), None)
def test_setting_existing_generator(self):
mock_generator = lambda: None
self.grammar.set_custom_generator("number", mock_generator)
eq_(self.grammar.get_custom_generator("number"), mock_generator)
def test_requesting_non_existing_generator(self):
assert_raises(GrammarError, self.grammar.get_custom_generator,
"non_existing")
def test_setting_non_existing_generator(self):
mock_generator = lambda: None
assert_raises(GrammarError, self.grammar.set_custom_generator,
"non_existing", mock_generator)
def test_adding_existing_grammar(self):
"""There can't be duplicate/overlapped parsers."""
mgr = EvaluableParseManager(self.symbol_table, Grammar(), es=Grammar())
assert_raises(GrammarError, mgr.add_parser, "es", Grammar())
class TestNativeVariableEvaluableParseManager(object):
"""
Tests for the :class:`EvaluableParseManager` with caching disabled.
"""
# A symbol table to be used across the tests:
symbol_table = SymbolTable(
"root", (
Bind("myint", NumberVariable("myint")),
Bind("mybool", BooleanVariable("mybool")),
Bind("mystring", StringVariable("mystring")),
Bind("mydate", DateVariable("mydate")),
Bind("mydate2", DateVariable("mydate2")),
Bind("mydatetime", DateTimeVariable("mydatetime")),
Bind("mydatetime2", DateTimeVariable("mydatetime2")),
Bind("myset", SetVariable("myset")),
Bind("myintbis", NumberVariable("myint")),
), SymbolTable(
"sub",
(Bind("myint", NumberVariable("sub.myint")), ),
),
SymbolTable("const", (
Bind("hello", String("hello")),
Bind("li", String("li")),
)))
mgr = EvaluableParseManager(
symbol_table, Grammar(belongs_to='in', is_subset='is subset of'))
def test_variable_values_myint(self):
ok_(self.mgr.parse('myint > 0')({'myint': 1}))
ok_(not self.mgr.parse('myint > 0')({'myint': 0}))
ok_(self.mgr.parse('myint == 1')({'myint': 1}))
ok_(not self.mgr.parse('myint == 1')({'myint': 0}))
ok_(self.mgr.parse('myint < 1')({'myint': 0}))
ok_(not self.mgr.parse('myint < 0')({'myint': 2}))
def test_variable_values_mybool(self):
ok_(self.mgr.parse('mybool')({'mybool': True}))
ok_(not self.mgr.parse('mybool')({'mybool': False}))
ok_(self.mgr.parse('mybool > 0')({'mybool': True}))
ok_(not self.mgr.parse('mybool > 0')({'mybool': False}))
ok_(self.mgr.parse('mybool == 1')({'mybool': True}))
ok_(not self.mgr.parse('mybool == 1')({'mybool': False}))
ok_(self.mgr.parse('mybool < 1')({'mybool': False}))
ok_(not self.mgr.parse('mybool < 0')({'mybool': True}))
def test_variable_values_mystring(self):
mystring = {'mystring': "hello"}
# equiality
ok_(self.mgr.parse('mystring == "hello"')(mystring))
ok_(not self.mgr.parse('mystring != "hello"')(mystring))
ok_(self.mgr.parse('mystring != "hell"')(mystring))
ok_(not self.mgr.parse('mystring == "hell"')(mystring))
# inequality
ok_(self.mgr.parse('mystring > "hella"')(mystring))
ok_(self.mgr.parse('mystring < "hellz"')(mystring))
ok_(self.mgr.parse('mystring < "hellz"')(mystring))
ok_(self.mgr.parse('mystring <= "hello"')(mystring))
ok_(self.mgr.parse('mystring >= "hello"')(mystring))
# inequality reversed
ok_(self.mgr.parse('"hella" < mystring')(mystring))
ok_(self.mgr.parse('"hellz" > mystring')(mystring))
ok_(self.mgr.parse('"hellz" > mystring')(mystring))
ok_(self.mgr.parse('"hello" >= mystring')(mystring))
ok_(self.mgr.parse('"hello" >= mystring')(mystring))
# bool
ok_(self.mgr.parse('mystring')(mystring))
ok_(not self.mgr.parse('~ mystring')(mystring))
ok_(not self.mgr.parse('mystring')({'mystring': ''}))
ok_(self.mgr.parse('~ mystring')({'mystring': ''}))
# membership
ok_(self.mgr.parse('mystring is subset of "zhelloy"')(mystring))
ok_(not self.mgr.parse('mystring is subset of "zhellai"')(mystring))
ok_(not self.mgr.parse('mystring is subset of "hello"')(mystring))
ok_(self.mgr.parse('mystring in "ahelloa"')(mystring))
ok_(self.mgr.parse('mystring in "hello"')(mystring))
ok_(not self.mgr.parse('mystring in "hola"')(mystring))
# inversed membership
ok_(self.mgr.parse('"he" in mystring')(mystring))
ok_(self.mgr.parse('"lo" in mystring')(mystring))
ok_(not self.mgr.parse('"li" in mystring')(mystring))
ok_(self.mgr.parse('"he" is subset of mystring')(mystring))
ok_(self.mgr.parse('"lo" is subset of mystring')(mystring))
ok_(not self.mgr.parse('"li" is subset of mystring')(mystring))
ok_(not self.mgr.parse(' "hello" is subset of mystring')(mystring))
# test to string
ok_(not self.mgr.parse('const:hello is subset of mystring')(mystring))
ok_(not self.mgr.parse('const:li in mystring')(mystring))
def test_variable_str(self):
eq_("%s" % NativeVariable('coucou'),
'Scop variable for coucou [NativeVariable]')
eq_("%s" % BooleanVariable('name'),
'Scop variable for name [BooleanVariable]')
eq_("%s" % NumberVariable('age'),
'Scop variable for age [NumberVariable]')
def test_variable_values_date(self):
mydate = {
'mydate': datetime.date(2017, 1, 15),
'mydate2': datetime.date(2017, 1, 17),
}
ok_(self.mgr.parse('mydate')(mydate))
ok_(self.mgr.parse('mydate < "2017-01-18"')(mydate))
ok_(self.mgr.parse('mydate < "18-01-2017"')(mydate))
ok_(not self.mgr.parse('mydate < "12-01-2017"')(mydate))
ok_(self.mgr.parse('mydate > "12-01-2017"')(mydate))
ok_(self.mgr.parse('mydate == "15-01-2017"')(mydate))
ok_(not self.mgr.parse('mydate == 1')(mydate))
ok_(not self.mgr.parse('mydate == "13-01-2017"')(mydate))
ok_(self.mgr.parse('mydate < mydate2')(mydate))
ok_(not self.mgr.parse('mydate > mydate2')(mydate))
ok_(not self.mgr.parse('mydate == mydate2')(mydate))
def test_variable_values_datetime(self):
mydatetime = {
'mydatetime': datetime.datetime(2017, 1, 15, 12, 25),
'mydatetime2': datetime.datetime(2017, 1, 17, 12, 23),
}
ok_(self.mgr.parse('mydatetime')(mydatetime))
ok_(self.mgr.parse('mydatetime < "2017-01-15 12:25:02"')(mydatetime))
ok_(self.mgr.parse('mydatetime > "2017-01-15 12:24:52"')(mydatetime))
ok_(self.mgr.parse('mydatetime < "2017-01-15 13:00:00"')(mydatetime))
ok_(self.mgr.parse('mydatetime < "18-01-2017 12:25:02"')(mydatetime))
ok_(self.mgr.parse('mydatetime < "18-01-2017 12:25:02"')(mydatetime))
ok_(not self.mgr.parse('mydatetime < "12-01-2017 12:25:02"')
(mydatetime))
ok_(self.mgr.parse('mydatetime > "12-01-2017 12:25:02"')(mydatetime))
ok_(self.mgr.parse('mydatetime == "15-01-2017 12:25:00"')(mydatetime))
ok_(not self.mgr.parse('mydatetime == 1')(mydatetime))
ok_(not self.mgr.parse('mydatetime == "13-01-2017 12:25:02"')
(mydatetime))
ok_(self.mgr.parse('mydatetime < mydatetime2')(mydatetime))
ok_(not self.mgr.parse('mydatetime > mydatetime2')(mydatetime))
ok_(not self.mgr.parse('mydatetime == mydatetime2')(mydatetime))
def test_variable_values_set(self):
myset = {
"myset": {1, 2, 3},
}
ok_(self.mgr.parse("1 is subset of myset ")(myset))
ok_(self.mgr.parse("{1, 2} is subset of myset ")(myset))
ok_(not self.mgr.parse("{1, 2, 3} is subset of myset ")(myset))
ok_(self.mgr.parse("1 in myset ")(myset))
ok_(self.mgr.parse("{1, 2} in myset ")(myset))
ok_(self.mgr.parse("{1, 2, 3} in myset ")(myset))
def test_variable_values_namespaced(self):
myints = {"myint": 1, "sub.myint": 4}
ok_(self.mgr.parse('sub:myint > 3')(myints))
ok_(self.mgr.parse('sub:myint < 5')(myints))
ok_(self.mgr.parse('sub:myint == 4')(myints))
ok_(self.mgr.parse('sub:myint > myint')(myints))
ok_(not self.mgr.parse('sub:myint < myint')(myints))
ok_(not self.mgr.parse('sub:myint == myint')(myints))
def test_format_date(self):
dt = DateVariable("mydate", '%d %m %y')
assert_raises(ValueError, dt._from_native_string, "2001-03-04")
eq_(dt._from_native_string("04 03 01"), datetime.date(2001, 3, 4))
def test_multi_formats(self):
dt = DateVariable("mydate", ['%d %m %Y', '%Y %m %d'])
assert_raises(ValueError, dt._from_native_string, "2001-03-04")
assert_raises(ValueError, dt._from_native_string, "04-03-2001")
eq_(dt._from_native_string("04 03 2001"), datetime.date(2001, 3, 4))
eq_(dt._from_native_string("2001 03 04"), datetime.date(2001, 3, 4))
def test_custom_tokens_against_trees(self):
"""
All the custom tokens in the grammar must be taken into account by the
parser.
To test that custom tokens are used, and in order to test many scenarios
writen few lines of code, we're going to convert operation nodes into
boolean expressions (using the relevant grammar), and then these
expressions will be parsed to check if the result is the original
tree.
"""
grammars = (
# A grammar that overrides all the default tokens:
Grammar(
**{
'not': "not",
'and': "and",
'xor': "xor",
'or': "or",
'eq': "equals",
'ne': "different-from",
'lt': "less-than",
'le': "less-equal",
'gt': "greater-than",
'ge': "greater-equal",
'belongs_to': "belongs-to",
'is_subset': "is-subset-of",
'set_start': "\\",
'set_end': "/",
'element_separator': ";",
'arguments_start': "[",
'arguments_end': "]",
'arguments_separator': ";",
'namespace_separator': ".",
}),
# Now let's try a grammar where some operators represent the
# initial characters of other operators:
Grammar(
**{
'eq': "is",
'ne': "isn't",
'lt': "is less than",
'gt': "is greater than",
'le': "is less than or equal to",
'ge': "is greater than or equal to",
'belongs_to': "is included in",
'is_subset': "is subset of",
}))
for grammar_num in range(len(grammars)):
grammar = grammars[grammar_num]
parser = ConvertibleParser(grammar)
convert_to_string = StringConverter(grammar)
for operation in self.expressions.values():
expression = convert_to_string(operation)
# Using a Nose test generator:
def check():
new_operation = parser(expression).root_node
eq_(
operation, new_operation,
u'Original operation: %s --- Returned operation: %s' %
(repr(operation), repr(new_operation)))
check.description = (u"The following expression is valid in "
u"the grammar #%s: %s" %
(grammar_num, expression))
yield check
def setUp(self):
self.grammar = Grammar()
class TestDefaultGrammar(BaseGrammarTest):
"""
Tests for the parser of the default/generic grammar.
"""
grammar = Grammar()
expressions = {
# Literals-only expressions:
' "a string" == 245 ':
Equal(String("a string"), Number(245)),
' 2 > 5 ':
GreaterThan(Number(2), Number(5)),
# Identifiers-only expressions:
' today > yesterday ':
GreaterThan(PlaceholderVariable("today"),
PlaceholderVariable("yesterday")),
'time:today > time:yesterday ':
GreaterThan(PlaceholderVariable("today", ("time", )),
PlaceholderVariable("yesterday", ("time", ))),
# Literals + non-literal expressions:
'today > "1999-06-01"':
GreaterThan(PlaceholderVariable("today"), String("1999-06-01")),
'get_parents("Gustavo") == {"Liliana", "Carlos"}':
Equal(PlaceholderFunction("get_parents", None, String("Gustavo")),
Set(String("Liliana"), String("Carlos"))),
# Set-specific operations:
u'"hi" ∈ {"hi", "bye"}':
BelongsTo(String("hi"), Set(String("hi"), String("bye"))),
u'salutation ∈ {"hi", "bye"}':
BelongsTo(PlaceholderVariable("salutation"),
Set(String("hi"), String("bye"))),
u'"good morning" ∈ salutations':
BelongsTo(String("good morning"), PlaceholderVariable("salutations")),
u'relatives:sister ∈ relatives:siblings':
BelongsTo(PlaceholderVariable("sister", ("relatives", )),
PlaceholderVariable("siblings", ("relatives", ))),
u'{"hi", "bye"} ⊂ {"hello", "hi", "bye"}':
IsSubset(Set(String("hi"), String("bye")),
Set(String("hello"), String("hi"), String("bye"))),
u'salutations ⊂ {"morning", "hi", "bye", "later"}':
IsSubset(
PlaceholderVariable("salutations"),
Set(String("morning"), String("hi"), String("bye"),
String("later"))),
u'{"hi", "bye"} ⊂ salutations':
IsSubset(Set(String("hi"), String("bye")),
PlaceholderVariable("salutations")),
u'relatives:siblings ⊂ relatives:everyone':
IsSubset(PlaceholderVariable("siblings", ("relatives", )),
PlaceholderVariable("everyone", ("relatives", ))),
# Relational operations:
'now == today:time':
Equal(PlaceholderVariable("now"),
PlaceholderVariable("time", ("today", ))),
'3.1416 == Pi':
Equal(Number(3.1416), PlaceholderVariable("pi")),
'Pi == 3.1416':
Equal(PlaceholderVariable("pi"), Number(3.1416)),
'days:today != days:yesterday':
NotEqual(PlaceholderVariable("today", ("days", )),
PlaceholderVariable("yesterday", ("days", ))),
'3.00 != Pi':
NotEqual(Number(3.0), PlaceholderVariable("pi")),
'Pi != 3.00':
Equal(PlaceholderVariable("pi"), Number(3.00)),
'days:today > days:yesterday':
GreaterThan(PlaceholderVariable("today", ("days", )),
PlaceholderVariable("yesterday", ("days", ))),
'3.00 > Pi':
GreaterThan(Number(3.0), PlaceholderVariable("pi")),
'Pi > 3.00':
GreaterThan(PlaceholderVariable("pi"), Number(3.00)),
'days:today < days:yesterday':
LessThan(PlaceholderVariable("today", ("days", )),
PlaceholderVariable("yesterday", ("days", ))),
'3.00 < Pi':
LessThan(Number(3.0), PlaceholderVariable("pi")),
'Pi < 3.00':
LessThan(PlaceholderVariable("pi"), Number(3.00)),
'days:today >= days:yesterday':
GreaterEqual(PlaceholderVariable("today", ("days", )),
PlaceholderVariable("yesterday", ("days", ))),
'3.00 >= Pi':
GreaterEqual(Number(3.0), PlaceholderVariable("pi")),
'Pi >= 3.00':
GreaterEqual(PlaceholderVariable("pi"), Number(3.00)),
'days:today <= days:yesterday':
LessEqual(PlaceholderVariable("today", ("days", )),
PlaceholderVariable("yesterday", ("days", ))),
'3.00 <= Pi':
LessEqual(Number(3.0), PlaceholderVariable("pi")),
'Pi <= 3.00':
LessEqual(PlaceholderVariable("pi"), Number(3.00)),
# Logical connectives:
'~today_will_rain()':
Not(PlaceholderFunction("today_will_rain")),
'~(today_will_rain())':
Not(PlaceholderFunction("today_will_rain")),
'~ today_will_rain()':
Not(PlaceholderFunction("today_will_rain")),
'~~today_will_rain()':
Not(Not(PlaceholderFunction("today_will_rain"))),
'~ today_is_friday':
Not(PlaceholderVariable("today_is_friday")),
'~ time:days:today > "1999-06-01"':
Not(
GreaterThan(PlaceholderVariable("today", ("time", "days")),
String("1999-06-01"))),
'~ (time:days:today > "1999-06-01")':
Not(
GreaterThan(PlaceholderVariable("today", ("time", "days")),
String("1999-06-01"))),
'~ today_is_friday & today_will_rain':
And(Not(PlaceholderVariable("today_is_friday")),
PlaceholderVariable("today_will_rain")),
'~ today_is_friday & ~ today_will_rain':
And(Not(PlaceholderVariable("today_is_friday")),
Not(PlaceholderVariable("today_will_rain"))),
'today_is_friday & ~ today_will_rain':
And(PlaceholderVariable("today_is_friday"),
Not(PlaceholderVariable("today_will_rain"))),
'~ (today_is_friday & today_will_rain)':
Not(
And(PlaceholderVariable("today_is_friday"),
PlaceholderVariable("today_will_rain"))),
'~ X > 3 & Z < X':
And(Not(GreaterThan(PlaceholderVariable("X"), Number(3))),
LessThan(PlaceholderVariable("Z"), PlaceholderVariable("X"))),
'~ X > 3 & ~ Z < X':
And(Not(GreaterThan(PlaceholderVariable("X"), Number(3))),
Not(LessThan(PlaceholderVariable("Z"), PlaceholderVariable("X")))),
'X > 3 & ~ Z < X':
And(GreaterThan(PlaceholderVariable("X"), Number(3)),
Not(LessThan(PlaceholderVariable("Z"), PlaceholderVariable("X")))),
'~ (X > 3 & Z < X)':
Not(
And(GreaterThan(PlaceholderVariable("X"), Number(3)),
LessThan(PlaceholderVariable("Z"), PlaceholderVariable("X")))),
'~ (X > 3 ^ Z < X)':
Not(
Xor(GreaterThan(PlaceholderVariable("X"), Number(3)),
LessThan(PlaceholderVariable("Z"), PlaceholderVariable("X")))),
'~ (X > 3 | Z < X)':
Not(
Or(GreaterThan(PlaceholderVariable("X"), Number(3)),
LessThan(PlaceholderVariable("Z"), PlaceholderVariable("X")))),
'today_will_rain() & Pi > 3.0':
And(PlaceholderFunction("today_will_rain"),
GreaterThan(PlaceholderVariable("Pi"), Number(3.0))),
'weather:today_rains & Pi > e & today_is_monday()':
And(
PlaceholderVariable("today_rains", ("weather", )),
And(
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e")),
PlaceholderFunction("today_is_monday"))),
'weather:today_rains & (Pi > e & today_is_monday())':
And(
PlaceholderVariable("today_rains", ("weather", )),
And(
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e")),
PlaceholderFunction("today_is_monday"))),
'(weather:today_rains & Pi > e) & today_is_monday()':
And(
And(
PlaceholderVariable("today_rains", ("weather", )),
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e"))),
PlaceholderFunction("today_is_monday")),
'today_will_rain() ^ Pi > 3.0':
Xor(PlaceholderFunction("today_will_rain"),
GreaterThan(PlaceholderVariable("Pi"), Number(3.0))),
'weather:today_rains ^ Pi > e ^ today_is_monday()':
Xor(
PlaceholderVariable("today_rains", ("weather", )),
Xor(
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e")),
PlaceholderFunction("today_is_monday"))),
'weather:today_rains ^ (Pi > e ^ today_is_monday())':
Xor(
PlaceholderVariable("today_rains", ("weather", )),
Xor(
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e")),
PlaceholderFunction("today_is_monday"))),
'(weather:today_rains ^ Pi > e) ^ today_is_monday()':
Xor(
Xor(
PlaceholderVariable("today_rains", ("weather", )),
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e"))),
PlaceholderFunction("today_is_monday")),
'today_will_rain() | Pi > 3.0':
Or(PlaceholderFunction("today_will_rain"),
GreaterThan(PlaceholderVariable("Pi"), Number(3.0))),
'weather:today_rains | Pi > e | today_is_monday()':
Or(
PlaceholderVariable("today_rains", ("weather", )),
Or(
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e")),
PlaceholderFunction("today_is_monday"))),
'weather:today_rains | (Pi > e | today_is_monday())':
Or(
PlaceholderVariable("today_rains", ("weather", )),
Or(
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e")),
PlaceholderFunction("today_is_monday"))),
'(weather:today_rains | Pi > e) | today_is_monday()':
Or(
Or(
PlaceholderVariable("today_rains", ("weather", )),
GreaterThan(PlaceholderVariable("Pi"),
PlaceholderVariable("e"))),
PlaceholderFunction("today_is_monday")),
# Checking operator precedence rules:
'today == "monday" & yesterday != "sunday" ^ Y > 0':
Xor(
And(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'(today == "monday" & yesterday != "sunday") ^ Y > 0':
Xor(
And(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'today == "monday" & (yesterday != "sunday" ^ Y > 0)':
And(
Equal(PlaceholderVariable("today"), String("monday")),
Xor(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'today == "monday" ^ yesterday != "sunday" & Y > 0':
Xor(
Equal(PlaceholderVariable("today"), String("monday")),
And(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'today == "monday" ^ (yesterday != "sunday" & Y > 0)':
Xor(
Equal(PlaceholderVariable("today"), String("monday")),
And(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'(today == "monday" ^ yesterday != "sunday") & Y > 0':
And(
Xor(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'today == "monday" & yesterday != "sunday" | Y > 0':
Or(
And(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'(today == "monday" & yesterday != "sunday") | Y > 0':
Or(
And(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'today == "monday" & (yesterday != "sunday" | Y > 0)':
And(
Equal(PlaceholderVariable("today"), String("monday")),
Or(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'today == "monday" | yesterday != "sunday" & Y > 0':
Or(
Equal(PlaceholderVariable("today"), String("monday")),
And(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'today == "monday" | (yesterday != "sunday" & Y > 0)':
Or(
Equal(PlaceholderVariable("today"), String("monday")),
And(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'(today == "monday" | yesterday != "sunday") & Y > 0':
And(
Or(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'today == "monday" ^ yesterday != "sunday" | Y > 0':
Or(
Xor(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'(today == "monday" ^ yesterday != "sunday") | Y > 0':
Or(
Xor(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'today == "monday" ^ (yesterday != "sunday" | Y > 0)':
Xor(
Equal(PlaceholderVariable("today"), String("monday")),
Or(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'today == "monday" | yesterday != "sunday" ^ Y > 0':
Or(
Equal(PlaceholderVariable("today"), String("monday")),
Xor(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'today == "monday" | (yesterday != "sunday" ^ Y > 0)':
Or(
Equal(PlaceholderVariable("today"), String("monday")),
Xor(NotEqual(PlaceholderVariable("yesterday"), String("sunday")),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
),
'(today == "monday" | yesterday != "sunday") ^ Y > 0':
Xor(
Or(Equal(PlaceholderVariable("today"), String("monday")),
NotEqual(PlaceholderVariable("yesterday"), String("sunday"))),
GreaterThan(PlaceholderVariable("Y"), Number(0))),
'W == 0 & X != 1 ^ Y > 2 | Z < 3':
Or(
Xor(
And(Equal(PlaceholderVariable("W"), Number(0)),
NotEqual(PlaceholderVariable("X"), Number(1))),
GreaterThan(PlaceholderVariable("Y"), Number(2))),
LessThan(PlaceholderVariable("Z"), Number(3))),
'W == 0 | X != 1 ^ Y > 2 & Z < 3':
Or(
Equal(PlaceholderVariable("W"), Number(0)),
Xor(
NotEqual(PlaceholderVariable("X"), Number(1)),
And(GreaterThan(PlaceholderVariable("Y"), Number(2)),
LessThan(PlaceholderVariable("Z"), Number(3))),
),
),
'W == 0 | (X != 1 ^ (Y > 2 & Z < 3))':
Or(
Equal(PlaceholderVariable("W"), Number(0)),
Xor(
NotEqual(PlaceholderVariable("X"), Number(1)),
And(GreaterThan(PlaceholderVariable("Y"), Number(2)),
LessThan(PlaceholderVariable("Z"), Number(3))),
),
),
'(W == 0 | X != 1) ^ (Y > 2 & Z < 3)':
Xor(
Or(Equal(PlaceholderVariable("W"), Number(0)),
NotEqual(PlaceholderVariable("X"), Number(1))),
And(GreaterThan(PlaceholderVariable("Y"), Number(2)),
LessThan(PlaceholderVariable("Z"), Number(3)))),
'(W == 0 | X != 1 ^ Y > 2) & Z < 3':
And(
Or(
Equal(PlaceholderVariable("W"), Number(0)),
Xor(
NotEqual(PlaceholderVariable("X"), Number(1)),
GreaterThan(PlaceholderVariable("Y"), Number(2)),
)), LessThan(PlaceholderVariable("Z"), Number(3))),
'((W == 0 | X != 1) ^ Y > 2) & Z < 3':
And(
Xor(
Or(Equal(PlaceholderVariable("W"), Number(0)),
NotEqual(PlaceholderVariable("X"), Number(1))),
GreaterThan(PlaceholderVariable("Y"), Number(2))),
LessThan(PlaceholderVariable("Z"), Number(3))),
'(W == 0 | (X != 1 ^ Y > 2)) & Z < 3':
And(
Or(
Equal(PlaceholderVariable("W"), Number(0)),
Xor(NotEqual(PlaceholderVariable("X"), Number(1)),
GreaterThan(PlaceholderVariable("Y"), Number(2)))),
LessThan(PlaceholderVariable("Z"), Number(3))),
'W == 0 | (X != 1 ^ Y > 2 & Z < 3)':
Or(
Equal(PlaceholderVariable("W"), Number(0)),
Xor(
NotEqual(PlaceholderVariable("X"), Number(1)),
And(GreaterThan(PlaceholderVariable("Y"), Number(2)),
LessThan(PlaceholderVariable("Z"), Number(3))))),
'W == 0 | (X != 1 ^ (Y > 2 & Z < 3))':
Or(
Equal(PlaceholderVariable("W"), Number(0)),
Xor(
NotEqual(PlaceholderVariable("X"), Number(1)),
And(GreaterThan(PlaceholderVariable("Y"), Number(2)),
LessThan(PlaceholderVariable("Z"), Number(3))))),
'W == 0 | ((X != 1 ^ Y > 2) & Z < 3)':
Or(
Equal(PlaceholderVariable("W"), Number(0)),
And(
Xor(
NotEqual(PlaceholderVariable("X"), Number(1)),
GreaterThan(PlaceholderVariable("Y"), Number(2)),
), LessThan(PlaceholderVariable("Z"), Number(3)))),
# Let's make sure whitespace doesn't change anything:
'''
variable == "hi"
''':
Equal(PlaceholderVariable("variable"), String("hi")),
'''
variable
==
"hi"
''':
Equal(PlaceholderVariable("variable"), String("hi")),
'''
\t variable == "hi"
&
\t today_is_thursday()
''':
And(Equal(PlaceholderVariable("variable"), String("hi")),
PlaceholderFunction("today_is_thursday")),
}
badformed_expressions = (
"",
" ",
"\t",
"\n",
"\r",
"== 3 4",
"3 4 ==",
)
single_operands = {
# ----- Strings
'"oneword"': String("oneword"),
'"double quotes"': String("double quotes"),
"'single quotes'": String("single quotes"),
"''": String(""),
'""': String(""),
"'something with \"quotes\"'": String('something with "quotes"'),
'"something with \'quotes\'"': String("something with 'quotes'"),
u'"áéíóúñçÁÉÍÓÚÑÇ"': String(u"áéíóúñçÁÉÍÓÚÑÇ"),
u'"海納百川，有容乃大"': String(u"海納百川，有容乃大"),
u"'مقالة مختارة'": String(u"مقالة مختارة"),
u'"вільної енциклопедії"': String(u"вільної енциклопедії"),
# ----- Numbers
'1': Number(1),
'01': Number(1),
'5000': Number(5000),
'+3': Number(3),
'-3': Number(-3),
'2.34': Number(2.34),
'2.2': Number(2.2),
'+3.1416': Number(3.1416),
'-3.1416': Number(-3.1416),
'5,000': Number(5000),
'1,000,000.34': Number(1000000.34),
'+1,000,000.22': Number(1000000.22),
'-1,000,000.22': Number(-1000000.22),
# ----- Sets:
' {} ': Set(),
'{{}, {}}': Set(Set(), Set()),
'{1,000, 3.05}': Set(Number(1000), Number(3.05)),
'{1,234,567}': Set(Number(1234567)),
'{23,24,25}': Set(Number(23), Number(24), Number(25)),
'{100, 200, 300}': Set(Number(100), Number(200), Number(300)),
'{1, 2, {"orange", "apple"}, 3}': Set(
Number(1),
Number(2),
Number(3),
Set(String("orange"), String("apple"))
),
u'{"españa", {"caracas", {"las chimeneas", "el trigal"}}, "france"}': \
Set(
String(u"españa"),
String("france"),
Set(
String("caracas"),
Set(String("el trigal"), String("las chimeneas"))
),
),
'{var1, var2}': Set(PlaceholderVariable("var1"),
PlaceholderVariable("var2")),
'{var, "string"}': Set(PlaceholderVariable("var"), String("string")),
'{3, var, "string"}': Set(Number(3), String("string"),
PlaceholderVariable("var")),
'{varA, funcB()}': Set(PlaceholderVariable("varA"),
PlaceholderFunction("funcB")),
'{"hi", 3.1416, len({1, 2, 3, 5, 7}), var0}': Set(
String("hi"),
Number(3.1416),
PlaceholderFunction("len", (), Set(
Number(1), Number(2), Number(3), Number(5), Number(7))),
PlaceholderVariable("var0")
),
# ----- Variables:
'today': PlaceholderVariable("today"),
'camelCase': PlaceholderVariable("camelCase"),
'with_underscore': PlaceholderVariable("with_underscore"),
' v1 ': PlaceholderVariable("v1"),
'var1_here': PlaceholderVariable("var1_here"),
u'résumé': PlaceholderVariable(u"résumé"),
u'有容乃大': PlaceholderVariable(u"有容乃大"),
' spaces': PlaceholderVariable("spaces"),
'spaces ': PlaceholderVariable("spaces"),
' spaces ': PlaceholderVariable("spaces"),
'_protected_var': PlaceholderVariable("_protected_var"),
'__private_var': PlaceholderVariable("__private_var"),
'one_underscore': PlaceholderVariable("one_underscore"),
'two__underscores__here': PlaceholderVariable("two__underscores__here"),
'case_insensitive_var': PlaceholderVariable("CASE_INSENSITIVE_VAR"),
'CASE_INSENSITIVE_VAR': PlaceholderVariable("case_insensitive_var"),
'cAsE_iNsEnSiTiVe_VaR': PlaceholderVariable("CaSe_InSeNsItIvE_vAr"),
u'MAYÚSCULA_minúscula': PlaceholderVariable(u"mayúscula_MINÚSCULA"),
'ns:variable': PlaceholderVariable("variable", ("ns", )),
'ns0:ns1:variable': PlaceholderVariable("variable", ("ns0", "ns1")),
'ns0:ns1:ns2:variable': PlaceholderVariable("variable",
("ns0", "ns1", "ns2")),
'ns:sub_ns:variable': PlaceholderVariable("variable", ("ns", "sub_ns")),
u'ñŝ0:ñŝ1:variable': PlaceholderVariable("variable", (u"ñŝ0", u"ñŝ1")),
# ----- Functions:
'stop()': PlaceholderFunction("stop"),
'stop ()': PlaceholderFunction("stop"),
'stop( )': PlaceholderFunction("stop"),
'stop ( )': PlaceholderFunction("stop"),
'camelCase()': PlaceholderFunction("camelCase"),
'with_underscore()': PlaceholderFunction("with_underscore"),
' v1() ': PlaceholderFunction("v1"),
'var1_here()': PlaceholderFunction("var1_here"),
u'résumé()': PlaceholderFunction(u"résumé"),
u'有容乃大()': PlaceholderFunction(u"有容乃大"),
' spaces()': PlaceholderFunction("spaces"),
'spaces() ': PlaceholderFunction("spaces"),
' spaces() ': PlaceholderFunction("spaces"),
'_protected_function()': PlaceholderFunction("_protected_function"),
'__private_function()': PlaceholderFunction("__private_function"),
'one_underscore()': PlaceholderFunction("one_underscore"),
'two__underscores__()': PlaceholderFunction("two__underscores__"),
'case_insensitive_func()': PlaceholderFunction("CASE_INSENSITIVE_FUNC"),
'CASE_INSENSITIVE_FUNC()': PlaceholderFunction("case_insensitive_func"),
'cAsE_iNsEnSiTiVe_FuNc()': PlaceholderFunction("CaSe_InSeNsItIvE_fUnC"),
u'MAYÚSCULA_minúscula()': PlaceholderFunction(u"mayúscula_MINÚSCULA"),
'func("argument")': PlaceholderFunction("func", (), String("argument")),
'function("arg1", variable, 3)': PlaceholderFunction("function",
(),
String("arg1"),
PlaceholderVariable("variable"),
Number(3),
),
'function("arg1", ns:variable, 3)': PlaceholderFunction(
"function",
(),
String("arg1"),
PlaceholderVariable("variable", ("ns", )),
Number(3),
),
'ns:function("arg1", ns:variable, 3)': PlaceholderFunction(
"function",
("ns", ),
String("arg1"),
PlaceholderVariable("variable", ("ns", )),
Number(3),
),
'function("arg1", ns:sub_function(), 3.0)': PlaceholderFunction(
"function",
(),
String("arg1"),
PlaceholderFunction("sub_function", ("ns", )),
Number(3),
),
'ns:function("arg1", ns:sub_function(), 3.0)': PlaceholderFunction(
"function",
("ns", ),
String("arg1"),
PlaceholderFunction("sub_function", ("ns", )),
Number(3),
),
'ns:function()': PlaceholderFunction("function", ("ns", )),
'ns0:ns1:function()': PlaceholderFunction("function", ("ns0", "ns1")),
'ns0:ns1:ns2:function()': PlaceholderFunction("function",
("ns0", "ns1", "ns2")),
'ns:sub_ns:function("with argument")': PlaceholderFunction("function",
("ns", "sub_ns"),
String("with argument"),
),
u'ñŝ:ñś1:function()': PlaceholderFunction("function", (u"ñŝ", u"ñś1")),
u'ñŝ:ñś1:función()': PlaceholderFunction(u"función", (u"ñŝ", u"ñś1")),
}
invalid_operands = (
# Invalid strings:
'\'mixed quotes"',
# Invalid numbers:
"3 . 1416",
"3. 1416",
"3 .1416",
"1,00",
"12.4,500,000",
# Invalid variables:
"dashes-here-cant-you-see-them",
"1st_variable",
"25th_variable",
"namespace-here:variable",
"1stnamespace:var",
"global:2ndlevel:var",
"namespace:1st_variable",
# Invalid functions:
"func(",
"func)",
"func)(",
"func[]",
"func{}",
"func(,)",
"func(arg1, )",
"func(, arg2)",
"func(, arg2, )",
"function-name()",
"1st_function()",
"25th_function()",
"namespace-here:function()",
"1stnamespace:function()",
"global:2ndlevel:function()",
"namespace:1st_function()",
"foo( bad-namespace:baz() )",
# Invalid sets:
"[]",
"{]",
"[}",
"}{",
"{key: 'value'}",
"[element1, element2, element3]",
"{element 1, element 2}",
"{element1; element2; element3}",
"{element == 'string'}",
# Default operators:
"==",
"!=",
"<",
">",
"<=",
">=",
"~",
"&",
"^",
"|",
u"∈",
u"⊂",
# Miscellaneous:
"-",
"this is definitely not an operand",
)
def test_custom_tokens_against_trees(self):
"""
All the custom tokens in the grammar must be taken into account by the
parser.
To test that custom tokens are used, and in order to test many scenarios
writen few lines of code, we're going to convert operation nodes into
boolean expressions (using the relevant grammar), and then these
expressions will be parsed to check if the result is the original
tree.
"""
grammars = (
# A grammar that overrides all the default tokens:
Grammar(
**{
'not': "not",
'and': "and",
'xor': "xor",
'or': "or",
'eq': "equals",
'ne': "different-from",
'lt': "less-than",
'le': "less-equal",
'gt': "greater-than",
'ge': "greater-equal",
'belongs_to': "belongs-to",
'is_subset': "is-subset-of",
'set_start': "\\",
'set_end': "/",
'element_separator': ";",
'arguments_start': "[",
'arguments_end': "]",
'arguments_separator': ";",
'namespace_separator': ".",
}),
# Now let's try a grammar where some operators represent the
# initial characters of other operators:
Grammar(
**{
'eq': "is",
'ne': "isn't",
'lt': "is less than",
'gt': "is greater than",
'le': "is less than or equal to",
'ge': "is greater than or equal to",
'belongs_to': "is included in",
'is_subset': "is subset of",
}))
for grammar_num in range(len(grammars)):
grammar = grammars[grammar_num]
parser = ConvertibleParser(grammar)
convert_to_string = StringConverter(grammar)
for operation in self.expressions.values():
expression = convert_to_string(operation)
# Using a Nose test generator:
def check():
new_operation = parser(expression).root_node
eq_(
operation, new_operation,
u'Original operation: %s --- Returned operation: %s' %
(repr(operation), repr(new_operation)))
check.description = (u"The following expression is valid in "
u"the grammar #%s: %s" %
(grammar_num, expression))
yield check