def test_array_simple4(self):
input_data = "{1, 2,3}"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
l_curly_token = Token(TokenType.L_CURLY, lexer.STRING_INPUT_FILE, 1, 1)
number_token1 = Token(TokenType.NUMBER,
lexer.STRING_INPUT_FILE,
1,
2,
value='1')
number_token2 = Token(TokenType.NUMBER,
lexer.STRING_INPUT_FILE,
1,
5,
value='2')
number_token3 = Token(TokenType.NUMBER,
lexer.STRING_INPUT_FILE,
1,
7,
value='3')
r_curly_token = Token(TokenType.R_CURLY, lexer.STRING_INPUT_FILE, 1, 8)
constant1 = Constant(number_token1)
constant2 = Constant(number_token2)
constant3 = Constant(number_token3)
array = Array(l_curly_token, [constant1, constant2, constant3],
r_curly_token)
self.assertEqual([array], ast)
def test_assignment2(self):
input_data = "a=1337;"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
left_token = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
1,
value='a')
equals_token = Token(TokenType.EQUALS, lexer.STRING_INPUT_FILE, 1, 2)
right_token = Token(TokenType.NUMBER,
lexer.STRING_INPUT_FILE,
1,
3,
value='1337')
semicolon_token = Token(TokenType.SEMICOLON, lexer.STRING_INPUT_FILE,
1, 7)
left_ast = Identifier(left_token)
right_ast = Constant(right_token)
assignment = Assignment(left_ast, equals_token, right_ast,
semicolon_token)
self.assertEqual([assignment], ast)
def _process_macro_usage(self):
"""
Processes one macro usage, e.g., "ADDON" or "EGVAR(main,variable)", replaces the old macro directive with the
expanded version and updates index accordingly.
:return: list of tokens - the expanded macro
"""
start_index = self.index # needed to replace the macro tokens we are expanding
macro_key = self.expect(TokenType.WORD).value
define = self.defines[
macro_key] # lookup the definition of the macro we are about to expand
if define.has_args():
# skip all tokens that belong to the macro, including the arguments and parenthesis
self.expect_next(TokenType.L_ROUND)
self.next()
unclosed_l_rounds = 1
while unclosed_l_rounds != 0:
if self.token().token_type == TokenType.L_ROUND:
unclosed_l_rounds += 1
elif self.token().token_type == TokenType.R_ROUND:
unclosed_l_rounds -= 1
self.index += 1
# expand the entire macro and insert it into the tokens + update index
expanded_macro = self._expand_macro(
self.tokens[start_index:self.index])
self.tokens = self.tokens[:
start_index] + expanded_macro + self.tokens[
self.index:]
self.index = start_index + len(expanded_macro)
else:
# macro consists of only 1 token, expand it and insert into tokens + update index
expanded_macro = self._expand_macro([self.token()])
if self.index > 0:
previous_token = self.tokens[self.index - 1]
if previous_token.token_type == TokenType.HASH:
# special case of stringify, e.g., #define QUOTE(var) #var -> QUOTE(hello) -> "hello"
l_quote = Token(TokenType.DOUBLE_QUOTES,
previous_token.file_path,
previous_token.line_no,
previous_token.line_pos)
r_quote = Token(TokenType.DOUBLE_QUOTES,
previous_token.file_path,
previous_token.line_no,
previous_token.line_pos)
expanded_macro = [l_quote] + expanded_macro + [r_quote]
self.index -= 1
del self.tokens[self.index]
start_index -= 1
self.tokens = self.tokens[:
start_index] + expanded_macro + self.tokens[
start_index + 1:]
self.index = self.index + len(expanded_macro)
return expanded_macro
def test_string_literal2(self):
input_data = 'hello world'
expected = [
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
1, 'hello'),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 6),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
7, 'world')
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_array_empty(self):
input_data = "{}"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
l_curly_token = Token(TokenType.L_CURLY, lexer.STRING_INPUT_FILE, 1, 1)
r_curly_token = Token(TokenType.R_CURLY, lexer.STRING_INPUT_FILE, 1, 2)
array = Array(l_curly_token, [], r_curly_token)
self.assertEqual([array], ast)
def test_number_literal4(self):
input_data = '-12.34'
expected = [
Token(TokenType.NUMBER, armaclassparser.lexer.STRING_INPUT_FILE, 1,
1, '-12.34')
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_string_literal3(self):
input_data = 'hello1234!'
expected = [
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
1, 'hello1234!')
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_include_simple(self):
input_data = '#include'
expected = [
Token(TokenType.KEYWORD_INCLUDE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 1)
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_class_simple(self):
input_data = 'class'
expected = [
Token(TokenType.KEYWORD_CLASS,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 1)
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_array_simple5(self):
input_data = "array[] = {};"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
name = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
1,
value='array')
l_square_token = Token(TokenType.L_SQUARE, lexer.STRING_INPUT_FILE, 1,
6)
r_square_token = Token(TokenType.R_SQUARE, lexer.STRING_INPUT_FILE, 1,
7)
identifier = Identifier(name)
array_declaration = ArrayDeclaration(identifier, l_square_token,
r_square_token)
equals_token = Token(TokenType.EQUALS, lexer.STRING_INPUT_FILE, 1, 9)
l_curly_token = Token(TokenType.L_CURLY, lexer.STRING_INPUT_FILE, 1,
11)
r_curly_token = Token(TokenType.R_CURLY, lexer.STRING_INPUT_FILE, 1,
12)
array = Array(l_curly_token, [], r_curly_token)
semicolon_token = Token(TokenType.SEMICOLON, lexer.STRING_INPUT_FILE,
1, 13)
assignment = Assignment(array_declaration, equals_token, array,
semicolon_token)
self.assertEqual([assignment], ast)
def test_array_simple2(self):
input_data = "{ 1 }"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
l_curly_token = Token(TokenType.L_CURLY, lexer.STRING_INPUT_FILE, 1, 1)
number_token = Token(TokenType.NUMBER,
lexer.STRING_INPUT_FILE,
1,
3,
value='2')
r_curly_token = Token(TokenType.R_CURLY, lexer.STRING_INPUT_FILE, 1, 5)
constant = Constant(number_token)
array = Array(l_curly_token, [constant], r_curly_token)
self.assertEqual([array], ast)
def test_assignment4(self):
input_data = 'author = "Schwaggot";'
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
left_token = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
1,
value='author')
equals_token = Token(TokenType.EQUALS, lexer.STRING_INPUT_FILE, 1, 8)
right_tokens = [
Token(TokenType.DOUBLE_QUOTES, lexer.STRING_INPUT_FILE, 1, 10),
Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
11,
value='Schwaggot'),
Token(TokenType.DOUBLE_QUOTES, lexer.STRING_INPUT_FILE, 1, 20)
]
semicolon_token = Token(TokenType.SEMICOLON, lexer.STRING_INPUT_FILE,
1, 21)
left_ast = Identifier(left_token)
right_ast = StringLiteral(right_tokens)
assignment = Assignment(left_ast, equals_token, right_ast,
semicolon_token)
self.assertEqual([assignment], ast)
def test_array_mixed1(self):
input_data = '{1, "hello"}'
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
l_curly_token = Token(TokenType.L_CURLY, lexer.STRING_INPUT_FILE, 1, 1)
number_token = Token(TokenType.NUMBER,
lexer.STRING_INPUT_FILE,
1,
2,
value='1')
l_double_quotes_token = Token(TokenType.DOUBLE_QUOTES,
lexer.STRING_INPUT_FILE, 1, 5)
string_token = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
6,
value='hello')
r_double_quotes_token = Token(TokenType.DOUBLE_QUOTES,
lexer.STRING_INPUT_FILE, 1, 11)
r_curly_token = Token(TokenType.R_CURLY, lexer.STRING_INPUT_FILE, 1,
12)
constant = Constant(number_token)
string_literal = StringLiteral(
[l_double_quotes_token, string_token, r_double_quotes_token])
array = Array(l_curly_token, [constant, string_literal], r_curly_token)
self.assertEqual([array], ast)
def test_assignment3(self):
input_data = "a = 'Hello';"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
left_token = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
1,
value='a')
equals_token = Token(TokenType.EQUALS, lexer.STRING_INPUT_FILE, 1, 3)
right_tokens = [
Token(TokenType.QUOTE, lexer.STRING_INPUT_FILE, 1, 5),
Token(TokenType.WORD, lexer.STRING_INPUT_FILE, 1, 6,
value='Hello'),
Token(TokenType.QUOTE, lexer.STRING_INPUT_FILE, 1, 11)
]
semicolon_token = Token(TokenType.SEMICOLON, lexer.STRING_INPUT_FILE,
1, 12)
left_ast = Identifier(left_token)
right_ast = StringLiteral(right_tokens)
assignment = Assignment(left_ast, equals_token, right_ast,
semicolon_token)
self.assertEqual([assignment], ast)
def test_array_declaration1(self):
input_data = "array[]"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
name = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
1,
value='array')
l_square_token = Token(TokenType.L_SQUARE, lexer.STRING_INPUT_FILE, 1,
6)
r_square_token = Token(TokenType.R_SQUARE, lexer.STRING_INPUT_FILE, 1,
7)
identifier = Identifier(name)
array_declaration = ArrayDeclaration(identifier, l_square_token,
r_square_token)
self.assertEqual([array_declaration], ast)
def test_identifier3(self):
input_data = "class0"
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(1, len(tokens))
parser = Parser(tokens)
ast = parser.parse()
self.assertEqual(1, len(ast))
expected_token = Token(TokenType.WORD,
lexer.STRING_INPUT_FILE,
1,
1,
value="class0")
identifier = Identifier(expected_token)
self.assertEqual([identifier], ast)
def test_include(self):
input_data = '#include "script_component.hpp"'
expected = [
Token(TokenType.KEYWORD_INCLUDE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 1),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 9),
Token(TokenType.DOUBLE_QUOTES,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 10),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
11, "script_component"),
Token(TokenType.DOT, armaclassparser.lexer.STRING_INPUT_FILE, 1,
27),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
28, "hpp"),
Token(TokenType.DOUBLE_QUOTES,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 31)
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_class(self):
input_data = 'class Foo {};'
expected = [
Token(TokenType.KEYWORD_CLASS,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 1),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 6),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
7, 'Foo'),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 10),
Token(TokenType.L_CURLY, armaclassparser.lexer.STRING_INPUT_FILE,
1, 11),
Token(TokenType.R_CURLY, armaclassparser.lexer.STRING_INPUT_FILE,
1, 12),
Token(TokenType.SEMICOLON, armaclassparser.lexer.STRING_INPUT_FILE,
1, 13)
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)
def test_multiline1(self):
input_data = '''#include "script_component.hpp"
class Foo {};'''
expected = [
Token(TokenType.KEYWORD_INCLUDE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 1),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 9),
Token(TokenType.DOUBLE_QUOTES,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 10),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
11, "script_component"),
Token(TokenType.DOT, armaclassparser.lexer.STRING_INPUT_FILE, 1,
27),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 1,
28, "hpp"),
Token(TokenType.DOUBLE_QUOTES,
armaclassparser.lexer.STRING_INPUT_FILE, 1, 31),
Token(TokenType.NEWLINE, armaclassparser.lexer.STRING_INPUT_FILE,
1, 32),
Token(TokenType.KEYWORD_CLASS,
armaclassparser.lexer.STRING_INPUT_FILE, 2, 1),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 2, 6),
Token(TokenType.WORD, armaclassparser.lexer.STRING_INPUT_FILE, 2,
7, 'Foo'),
Token(TokenType.WHITESPACE,
armaclassparser.lexer.STRING_INPUT_FILE, 2, 10),
Token(TokenType.L_CURLY, armaclassparser.lexer.STRING_INPUT_FILE,
2, 11),
Token(TokenType.R_CURLY, armaclassparser.lexer.STRING_INPUT_FILE,
2, 12),
Token(TokenType.SEMICOLON, armaclassparser.lexer.STRING_INPUT_FILE,
2, 13)
]
tokens = Lexer(input_data, lexer.STRING_INPUT_FILE).tokenize()
self.assertEqual(expected, tokens)