def __init__(self):
self.ast = Tree('metadata', [
Tree('metadata_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def test_multiple_assignments(parser: RecursiveDescentParser):
snippet = r"""
test() void {
let x int = n1 + n2
let y int = n3 + n4
}"""
expected = Tree('start', [Tree('function_declaration',
[Token('NAME', 'test'), Tree('function_parameters', []), Token('NAME', 'void'),
Tree('statements_block', [Tree('assignment', [Tree('variable_declaration',
[Token('LET', 'let'),
Token('NAME', 'x'), Tree('type',
[Token(
'NAME',
'int')])]),
Token('ASSIGNMENT_OPERATOR', '='),
Tree('additive_expression',
[Token('NAME', 'n1'),
Token('ADDITIVE_OPERATOR', '+'),
Token('NAME', 'n2')])]),
Tree('assignment', [Tree('variable_declaration',
[Token('LET', 'let'),
Token('NAME', 'y'), Tree('type',
[Token(
'NAME',
'int')])]),
Token('ASSIGNMENT_OPERATOR', '='),
Tree('additive_expression',
[Token('NAME', 'n3'),
Token('ADDITIVE_OPERATOR', '+'),
Token('NAME', 'n4')])])])])])
with io.StringIO(snippet) as f:
res, msg = compare_trees(expected, parser.parse(f))
assert res, msg
def test_Lark_ChargeConjugateReplacement_Visitor():
"""
A simple example usage of the ChargeConjugateReplacement implementation
of a Lark's Visitor, here replacing all particles in a 'decay' Tree
by their antiparticles.
"""
t = Tree(
"decay",
[
Tree("particle", [Token("LABEL", "D0")]),
Tree(
"decayline",
[
Tree("value", [Token("SIGNED_NUMBER", "1.0")]),
Tree("particle", [Token("LABEL", "K-")]),
Tree("particle", [Token("LABEL", "pi+")]),
Tree("model", [Token("MODEL_NAME", "PHSP")]),
],
),
],
)
ChargeConjugateReplacement().visit(t)
assert get_decay_mother_name(t) == "anti-D0"
assert get_final_state_particle_names(t.children[1]) == ["K+", "pi-"]
def __init__(self):
self.ast = Tree('output', [
Tree('output_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def test_reassignment(parser: RecursiveDescentParser):
snippet = r"""
test() void {
var x int = 1
x = 2
}"""
expected = Tree('start', [Tree('function_declaration',
[Token('NAME', 'test'),
Tree('function_parameters', []),
Token('NAME', 'void'),
Tree('statements_block', [
Tree('assignment', [
Tree('variable_declaration', [
Token('VAR', 'var'),
Token('NAME', 'x'),
Tree('type', [Token('NAME', 'int')])]),
Token('ASSIGNMENT_OPERATOR', '='),
Token('DEC_NUMBER', '1')]),
Tree('assignment', [
Token('NAME', 'x'),
Token('ASSIGNMENT_OPERATOR', '='),
Token('DEC_NUMBER', '2')])])])])
with io.StringIO(snippet) as f:
res, msg = compare_trees(expected, parser.parse(f))
assert res, msg
def process(self, stream: t.Iterator[lark.Token]) -> t.Iterator[Token]:
number_tokens: t.Optional[t.List[lark.Token]] = None
for token in stream:
try:
if not number_tokens:
number_tokens = []
self._english_parser.parse(number_tokens + [token])
number_tokens.append(token)
continue
except ValueError:
if number_tokens:
v = self._english_parser.parse(number_tokens)
yield Token.new_borrow_pos(
"UNSIGNED_INTEGER"
if isinstance(v, int) else "UNSIGNED_REAL",
str(self._english_parser.parse(number_tokens)),
number_tokens[0])
number_tokens = None
if not number_tokens:
try:
# Roman numerals must be at least 2 characters long,
# otherwise we can't use 'i' or 'x'...
if not len(str(token).strip()) < 2:
yield Token.new_borrow_pos(
"INTEGER", str(numeral.roman2int(token)), token)
continue
except ValueError:
pass
except NotImplementedError:
pass
yield token
def plus_zero_r(tree, entire):
""""
Performs x = x + 0
"""
if hasattr(tree, "parent") and tree.parent is not None:
grandparent = tree.parent
for e, child in enumerate(grandparent.children):
if child is tree:
treeindex = e
add = Tree("add", [])
zerotoken = Token("NUMBER", "0")
add.children = [tree, zerotoken]
grandparent.children[treeindex] = add
return tree, entire
else:
add = Tree("add", [])
zerotoken = Token("NUMBER", "0")
add.children = [tree, zerotoken]
entire = add
return tree, entire
def set_option(self, option_name, value):
tree = Tree('global_option_set', [
Token('GLOBAL_OPTION', option_name),
Tree('string', [Token('ESCAPED_STRING', f'"{value}"')]),
Token('DELIM', ';')
])
self.ast.children.append(tree)
def extension_n(self, children):
point = Tree(
"point",
[Token("NUMBER", children[0].value),
Token("OUTSIDE_CDS", "-")])
location = [Tree("location", [point])]
return Tree("inserted", [Tree("insert", location)])
def test_Lark_ChargeConjugateReplacement_Visitor_with_aliases():
"""
Example with a D0 decay specified via an alias (MyD0).
As such, it is necessary to state what the particle-antiparticle match is,
which in decay files would mean the following lines:
Alias MyD0 D0
Alias MyAnti-D0 anti-D0
ChargeConj MyD0 MyAnti-D0
A dictionary of matches should be passed to the Lark Visitor instance.
"""
t = Tree('decay', [
Tree('particle', [Token('LABEL', 'MyD0')]),
Tree('decayline', [
Tree('value', [Token('SIGNED_NUMBER', '1.0')]),
Tree('particle', [Token('LABEL', 'K-')]),
Tree('particle', [Token('LABEL', 'pi+')]),
Tree('model', [Token('MODEL_NAME', 'PHSP')])
])
])
dict_ChargeConj_defs = {'MyD0': 'MyAnti-D0'}
ChargeConjugateReplacement(charge_conj_defs=dict_ChargeConj_defs).visit(t)
assert get_decay_mother_name(t) == 'MyAnti-D0'
assert get_final_state_particle_names(t.children[1]) == ['K+', 'pi-']
def test_Lark_ChargeConjugateReplacement_Visitor_with_aliases():
"""
Example with a D0 decay specified via an alias (MyD0).
As such, it is necessary to state what the particle-antiparticle match is,
which in decay files would mean the following lines:
Alias MyD0 D0
Alias MyAnti-D0 anti-D0
ChargeConj MyD0 MyAnti-D0
A dictionary of matches should be passed to the Lark Visitor instance.
"""
t = Tree(
"decay",
[
Tree("particle", [Token("LABEL", "MyD0")]),
Tree(
"decayline",
[
Tree("value", [Token("SIGNED_NUMBER", "1.0")]),
Tree("particle", [Token("LABEL", "K-")]),
Tree("particle", [Token("LABEL", "pi+")]),
Tree("model", [Token("MODEL_NAME", "PHSP")]),
],
),
],
)
dict_ChargeConj_defs = {"MyD0": "MyAnti-D0"}
ChargeConjugateReplacement(charge_conj_defs=dict_ChargeConj_defs).visit(t)
assert get_decay_mother_name(t) == "MyAnti-D0"
assert get_final_state_particle_names(t.children[1]) == ["K+", "pi-"]
def ambiguous_literals(t: Token) -> Token:
"""Resolve a grammar ambiguity between identifiers and literals"""
if t.value == "true":
return Token("BOOL_LIT", t.value)
elif t.value == "false":
return Token("BOOL_LIT", t.value)
return t
def __init__(self):
self.option_block_token = 'PROCESS_INJECT_LOCAL_OPTION'
self.ast = Tree('process_inject', [
Tree('process_inject_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'EXECUTE_STATEMENT_OPTION'
self.ast = Tree('execute', [
Tree('execute_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'DNS_BEACON_LOCAL_OPTION'
self.ast = Tree('dns_beacon', [
Tree('dns_beacon_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'HTTPS_CERTIFICATE_LOCAL_OPTION'
self.ast = Tree('https_certificate', [
Tree('https_certificate_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'CODE_SIGNER_LOCAL_OPTION'
self.ast = Tree('code_signer', [
Tree('code_signer_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'STAGE_LOCAL_OPTION'
self.ast = Tree('stage', [
Tree('stage_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'HTTP_STAGER_LOCAL_OPTION'
self.ast = Tree('http_stager', [
Tree('http_stager_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def __init__(self):
self.option_block_token = 'POST_EX_LOCAL_OPTION'
self.ast = Tree('post_ex', [
Tree('post_ex_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def testEqualityVarName(self):
result = self.ast("var_x = 3")
expected = Tree(start, [
Tree(assignment,
[Token(NAME, 'var_x'),
Tree(string, [Token(WORD, '3')])])
])
self.assertEqual(expected, result)
def extension_c(self, children):
inserted = [Tree("insert", [Token("P_SEQUENCE", children[0]).value])]
if isinstance(children[1], Token):
inserted.append(
Tree("insert", [Token("P_SEQUENCE", children[1].value)]))
else:
inserted.append(Tree("insert", [Tree("location", [children[1]])]))
return Tree("inserted", inserted)
def __init__(self):
self.option_block_token = 'PROCESS_INJECT_LOCAL_OPTION'
self.ast = Tree('transform_x64', [
Tree('transform_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def process(self, stream):
token = None
for token in stream:
yield token
if token is None:
yield Token(self.token_type, "<EOF>")
else:
yield Token.new_borrow_pos(self.token_type, "<EOF>", token)
def __init__(self):
self.option_block_token = 'HTTP_CONFIG_LOCAL_OPTION'
self.ast = Tree('http_config', [
Tree('http_config_block', [
Token('BEGIN_CODE_BLOCK_DELIM', '{'),
Token('END_CODE_BLOCK_DELIM', '}')
])
])
def test_recursive_power_l():
# x ^ s(y) = x ^ y * x
example = Tree("suc", [
Tree("pow", [Token("WORD", "x"),
Tree("suc", [Token("NUMBER", "0")])])
])
example = label_parents(example)
pow_op = example.children[0]
cursor, entire = recursive_power_l(pow_op, example)
# Case of not root:
# suc
# pow <----
# x
# suc 0
# Expected result:
# suc
# mul
# pow
# x
# 0
# x
assert (
repr(entire) ==
"Tree(suc, [Tree(mul, [Tree(pow, [Token(WORD, 'x'), Token(NUMBER, '0')]), Token(WORD, 'x')])])"
)
example = Tree("pow",
[Token("WORD", "x"),
Tree("suc", [Token("NUMBER", "0")])])
example = label_parents(example)
pow_op = example
cursor, entire = recursive_power_l(pow_op, example)
# Case of root:
# pow <----
# x
# suc 0
# Expected result:
# mul
# pow
# x
# 0
# x
assert (
repr(entire) ==
"Tree(mul, [Tree(pow, [Token(WORD, 'x'), Token(NUMBER, '0')]), Token(WORD, 'x')])"
)
def format_size(size: Token) -> List[Token]:
for unit in NotionalUnitConverter.mapping.inv:
if abs(int(size.value) / unit) >= 1:
return [
Token(NotionalNumberConverter.name, size.value / unit),
Token(NotionalUnitConverter.name, unit)
]
else:
return [Token(NotionalNumberConverter.name, size.value)]
def test_power_of_one_l():
# 1 ^ x = 1
example = Tree("suc", [
Tree("pow", [Tree("suc", [Token("NUMBER", "0")]),
Token("WORD", "x")])
])
example = label_parents(example)
print("b")
print(example.pretty())
op = example.children[0]
cursor, entire = power_of_one_l(op, example)
print("b")
print(entire.pretty())
print(repr(entire))
# Case of not root
# suc
# pow
# suc 0
# x
# Expected result
# suc
# suc 0
assert repr(entire) == "Tree(suc, [Tree(suc, [Token(NUMBER, '0')])])"
example = Tree("pow",
[Tree("suc", [Token("NUMBER", "0")]),
Token("WORD", "x")])
example = label_parents(example)
print("b")
print(example.pretty())
cursor, entire = power_of_one_l(example, example)
print("b")
print(entire.pretty())
print(repr(entire))
# Case of root
# pow
# suc 0
# x
# Expected result
# suc 0
assert repr(entire) == "Tree(suc, [Token(NUMBER, '0')])"
def test_first_power_of_x_l():
# x ^ 1 = x
example = Tree("suc", [
Tree("pow", [Token("WORD", "x"),
Tree("suc", [Token("NUMBER", "0")])])
])
example = label_parents(example)
print("b")
print(example.pretty())
op = example.children[0]
cursor, entire = first_power_of_x_l(op, example)
print("b")
print(entire.pretty())
print(repr(entire))
# Case of not root
# suc
# pow
# x
# suc 0
# Expected results
# suc x
assert repr(entire) == "Tree(suc, [Token(WORD, 'x')])"
example = Tree("pow",
[Token("WORD", "x"),
Tree("suc", [Token("NUMBER", "0")])])
example = label_parents(example)
print("b")
print(example.pretty())
cursor, entire = first_power_of_x_l(example, example)
print("b")
print(entire)
print(repr(entire))
# Case of root
# pow
# x
# suc 0
# Expected result
# x
assert repr(entire) == "Token(WORD, 'x')"
def to_token(cls, name: str, obj: bool) -> Token:
if not isinstance(obj, bool):
raise TokenConversionError(
f"{cls.__qualname__} can format booleans.")
if obj:
return Token(name, cls.flag)
else:
return Token(name, "")