def identify_token(token):
"""
The function that converts a token from tokenize to a BaseType.
"""
if isinstance(token, (Comment, String)):
return token
if token == ' ':
return Space()
if token == '\t':
return Tab()
if token == '\\\n':
return BrokenEndOfLine()
if token in ('(', ')', '[', ']', '{', '}', ',', ';'):
return ParserKeyword(token)
if token in ('\n', '\r\n'):
return EndOfLine(token)
if token in ('true', 'false'):
return Boolean(token == 'true')
try:
return Number(int(token))
except ValueError:
pass
try:
return Number(float(token))
except ValueError:
pass
if token in PREPROCESSORS:
return Preprocessor(token)
if token.lower() in NAMESPACES:
return Namespace(token)
elif token.lower() in KEYWORDS:
return Keyword(token)
else:
return Variable(token)
def parse_switch(interpreter, code):
conditions = []
default_used = False
for statement in code.base_tokens:
base_tokens = statement.base_tokens
# evaluate all the base_tokens, trying to obtain their values
values = []
for token in base_tokens:
v = interpreter.value(token)
values.append(v)
if type(values[0]) != SwitchType:
interpreter.exception(
SQFParserError(
statement.position,
'Switch code can only start with "case" or "default"'))
if values[0].keyword == Keyword('default'):
if default_used:
interpreter.exception(
SQFParserError(
code.position,
'Switch code contains more than 1 `default`'))
default_used = True
assert (isinstance(values[0].result, Code))
conditions.append(('default', values[0].result))
else:
case_condition = values[0].result
if len(values) == 1:
conditions.append((case_condition, None))
else:
assert (len(values) == 3 and values[1] == Keyword(':'))
outcome_statement = values[2]
conditions.append((case_condition, outcome_statement))
return conditions
def test_with_comments(self):
# _x=2;/* the two
# the three
# the four
# */
# _x=3'
s = Statement([
Statement([
V('_x'),
Keyword('='),
N(2)], ending=';'),
Statement([
Statement([Comment('/* the two \n the three\n the four\n */'),
EndOfLine('\n'),
V('_x')]),
Keyword('='),
N(3)
])
])
s.set_position((1,1))
self.assertEqual(Keyword('='), s[1][1])
self.assertEqual((5, 3), s[1][1].position)
def __init__(self, action=None):
super().__init__(Code, Keyword('forEach'), Array, None, action)
import math
from sqf.common_expressions import TryCatchExpression, ForEachExpression, \
WhileDoExpression, ForFromToDoExpression, ForSpecDoExpression, SwitchDoExpression, \
IfThenSpecExpression, IfThenElseExpression, IfThenExpression, IfThenExitWithExpression
from sqf.types import Keyword, Namespace, Number, Array, Code, Type, Boolean, String, Nothing, Variable
from sqf.exceptions import SQFParserError
from sqf.keywords import OP_ARITHMETIC, OP_COMPARISON, OP_LOGICAL
from sqf.expressions import BinaryExpression, UnaryExpression
from sqf.interpreter_types import SwitchType
OP_OPERATIONS = {
# Arithmetic
Keyword('+'): lambda x, y: x + y,
Keyword('-'): lambda x, y: x - y,
Keyword('*'): lambda x, y: x * y,
Keyword('/'): lambda x, y: x / y,
Keyword('%'): lambda x, y: x % y,
Keyword('mod'): lambda x, y: x % y,
Keyword('^'): lambda x, y: x**y,
Keyword('max'): lambda x, y: max(x, y),
Keyword('floor'): lambda x: math.floor(x),
# Comparison
Keyword('=='): lambda x, y: x == y,
Keyword('!='): lambda x, y: x != y,
Keyword('<'): lambda x, y: x < y,
Keyword('>'): lambda x, y: x < y,
Keyword('<='): lambda x, y: x <= y,
Keyword('>='): lambda x, y: x >= y,
def test_keyword(self):
self.assertEqual(Keyword('forEach'), Keyword('foreach'))
def test_reservedtoken(self):
self.assertEqual('for', str(Keyword('for')))
def __init__(self, action=None):
super().__init__(IfType, Keyword('then'), Code, None, action)
def __init__(self):
super().__init__(Keyword('if'), Boolean, IfType, lambda v, i: v)
def __init__(self, action=None):
super().__init__(TryType, Keyword('catch'), Code, None, action)
def __init__(self):
super().__init__(Keyword('try'), Code, TryType, lambda v, i: v)
COMMON_EXPRESSIONS = [
CaseExpression(),
DefaultExpression(),
WithExpression(),
WithDoExpression(),
TryExpression(),
TryCatchExpression(),
ForEachExpression(),
WhileExpression(),
WhileDoExpression(),
ForExpression(),
ForFromExpression(),
ForFromToExpression(),
ForFromToStepExpression(),
ForFromToDoExpression(),
ForSpecExpression(),
ForSpecDoExpression(),
SwitchExpression(),
SwitchDoExpression(),
IfExpression(),
ElseExpression(),
IfThenSpecExpression(),
IfThenElseExpression(),
IfThenExpression(),
IfThenExitWithExpression(),
UnaryExpression(Keyword('params'), Array, Boolean,
lambda rhs_v, i: i.add_params(rhs_v)),
BinaryExpression(Type, Keyword('params'), Array, Boolean,
lambda lhs_v, rhs_v, i: i.add_params(rhs_v)),
]
def parse_block(all_tokens,
analyze_tokens,
start=0,
initial_lvls=None,
stop_statement='both',
defines=None):
if not initial_lvls:
initial_lvls = _LEVELS
if defines is None:
defines = defaultdict(dict)
lvls = initial_lvls.copy()
statements = []
tokens = []
i = start
if not all_tokens:
return Statement([]), 0
while i < len(all_tokens):
token = all_tokens[i]
# begin #ifdef controls
if lvls['ifdef'] and token in OPEN_PARENTHESIS:
lvls['ifdef_open_close'] += 1
stop = False
if token in (Preprocessor('#ifdef'), Preprocessor('#ifndef')):
stop = True
lvls['ifdef'] += 1
expression, size = parse_block(all_tokens,
_analyze_simple,
i + 1,
lvls,
stop_statement,
defines=defines)
lvls['ifdef'] -= 1
if lvls['ifdef'] == 0:
assert (isinstance(expression, IfDefStatement))
replacing_expression = parse_ifdef_block(
expression, defines, get_coord(all_tokens[:i - 1]))
new_all_tokens = sqf.base_type.get_all_tokens(
tokens + replacing_expression)
result, _ = parse_block(new_all_tokens,
analyze_tokens,
0,
None,
stop_statement,
defines=defines)
expression.prepend(tokens)
expression = IfDefResult(expression, result.tokens)
statements.append(expression)
len_expression = len(expression.get_all_tokens())
i += len_expression - len(tokens) - 1
tokens = []
else:
tokens.append(expression)
i += size + 1
# finish ifdef
elif is_finish_ifdef_condition(tokens, lvls) and (
is_end_statement(token, stop_statement)
or is_finish_ifdef_parenthesis(token, lvls)
) or lvls['ifdef'] > 1 and token == Preprocessor('#endif'):
if token != EndOfFile() and token not in CLOSE_PARENTHESIS:
tokens.append(token)
if_def = finish_ifdef(tokens, all_tokens, start, statements)
return if_def, i - start
# parse during ifdef
elif lvls['ifdef'] != 0:
stop = True
tokens.append(token)
# end ifdef controls
if lvls['ifdef'] and token in (STOP_KEYWORDS['single'] +
CLOSE_PARENTHESIS):
lvls['ifdef_open_close'] -= 1
if lvls['ifdef_open_close'] < 0:
lvls['ifdef_open_close'] = 0
if stop:
pass
# try to match a #defined and get the arguments
elif str(token) in defines: # is a define
stop, define_statement, arg_indexes = find_match_if_def(
all_tokens, i, defines, token)
if stop:
arg_number = len(define_statement.args)
extra_tokens_to_move = 1 + 2 * (
arg_number != 0) + 2 * arg_number - 1 * (arg_number != 0)
replaced_expression = all_tokens[i:i + extra_tokens_to_move]
# the `all_tokens` after replacement
replacing_expression = replace_in_expression(
define_statement.expression, define_statement.args,
arg_indexes, all_tokens)
new_all_tokens = all_tokens[:i - len(
tokens)] + tokens + replacing_expression + all_tokens[
i + extra_tokens_to_move:]
new_start = i - len(tokens)
expression, size = parse_block(new_all_tokens,
analyze_tokens,
new_start,
lvls,
stop_statement,
defines=defines)
# the all_tokens of the statement before replacement
original_tokens_taken = len(replaced_expression) - len(
replacing_expression) + size
original_tokens = all_tokens[i - len(tokens):i - len(tokens) +
original_tokens_taken]
if isinstance(expression, Statement):
expression = expression.content[0]
if type(original_tokens[-1]) in (EndOfLine, Comment,
EndOfFile):
del original_tokens[-1]
original_tokens_taken -= 1
expression = DefineResult(original_tokens, define_statement,
expression)
statements.append(expression)
i += original_tokens_taken - len(tokens) - 1
tokens = []
if stop:
pass
elif token == ParserKeyword('['):
lvls['[]'] += 1
expression, size = parse_block(all_tokens,
analyze_tokens,
i + 1,
lvls,
stop_statement='single',
defines=defines)
lvls['[]'] -= 1
tokens.append(expression)
i += size + 1
elif token == ParserKeyword('('):
lvls['()'] += 1
expression, size = parse_block(all_tokens,
analyze_tokens,
i + 1,
lvls,
stop_statement,
defines=defines)
lvls['()'] -= 1
tokens.append(expression)
i += size + 1
elif token == ParserKeyword('{'):
lvls['{}'] += 1
expression, size = parse_block(all_tokens,
analyze_tokens,
i + 1,
lvls,
stop_statement,
defines=defines)
lvls['{}'] -= 1
tokens.append(expression)
i += size + 1
elif token == ParserKeyword(']'):
if lvls['[]'] == 0:
raise SQFParenthesisError(
get_coord(all_tokens[:i]),
'Trying to close right parenthesis without them opened.')
if statements:
if isinstance(statements[0], DefineResult):
statements[0]._tokens = [
Array(
_analyze_array(statements[0]._tokens,
analyze_tokens, all_tokens[:i]))
]
return statements[0], i - start
else:
raise SQFParserError(
get_coord(all_tokens[:i]),
'A statement %s cannot be in an array' %
Statement(statements))
return Array(_analyze_array(tokens, analyze_tokens,
all_tokens[:i])), i - start
elif token == ParserKeyword(')'):
if lvls['()'] == 0:
raise SQFParenthesisError(
get_coord(all_tokens[:i]),
'Trying to close parenthesis without opened parenthesis.')
if tokens:
statements.append(analyze_tokens(tokens))
return Statement(statements, parenthesis=True), i - start
elif token == ParserKeyword('}'):
if lvls['{}'] == 0:
raise SQFParenthesisError(
get_coord(all_tokens[:i]),
'Trying to close brackets without opened brackets.')
if tokens:
statements.append(analyze_tokens(tokens))
return Code(statements), i - start
# end of statement when not in preprocessor states
elif all(lvls[lvl_type] == 0
for lvl_type in ('#define', '#include')) and is_end_statement(
token, stop_statement):
if type(token) != EndOfFile:
tokens.append(token)
if tokens:
statements.append(analyze_tokens(tokens))
tokens = []
elif token in (Preprocessor('#define'), Preprocessor('#include')):
# notice that `token` is ignored here. It will be picked up in the end
if tokens:
# a pre-processor starts a new statement
statements.append(analyze_tokens(tokens))
tokens = []
lvls[token.value] += 1
expression, size = parse_block(all_tokens,
analyze_tokens,
i + 1,
lvls,
stop_statement,
defines=defines)
lvls[token.value] -= 1
statements.append(expression)
i += size
elif token == Keyword('#') and lvls['#define'] != 0:
# The # sqf command is superseded by the preprocessor directive's stringification command
tokens.append(Preprocessor('#'))
elif type(token) in (EndOfLine, Comment, EndOfFile) and any(
lvls[x] != 0 for x in {'#define', '#include'}):
tokens.insert(
0,
all_tokens[start -
1]) # pick the token that triggered the statement
if tokens[0] == Preprocessor('#define'):
define_statement = _analyze_define(tokens)
defines[define_statement.variable_name][len(
define_statement.args)] = define_statement
statements.append(define_statement)
else:
statements.append(analyze_tokens(tokens))
return Statement(statements), i - start
elif type(token) != EndOfFile:
tokens.append(token)
i += 1
if is_finish_ifdef_condition(tokens, lvls):
return finish_ifdef(tokens, all_tokens, start, statements), i - start
for lvl_type in ('[]', '()', '{}', 'ifdef'):
if lvls[lvl_type] != 0:
message = 'Parenthesis "%s" not closed' % lvl_type[0]
if lvl_type == 'ifdef':
message = '#ifdef statement not closed'
raise SQFParenthesisError(get_coord(all_tokens[:start - 1]),
message)
if tokens:
statements.append(analyze_tokens(tokens))
return Statement(statements), i - start
def __init__(self):
super().__init__(Keyword('default'), Type, SwitchType, lambda v, i:
(self.keyword, v))
def __init__(self):
super().__init__(Keyword('with'), Namespace, WithType, lambda v, i: v)
def __init__(self, action=None):
super().__init__(SwitchType, Keyword('do'), Code, None, action)
def __init__(self, action=None):
if action is None:
action = lambda lhs, rhs, i: i.execute_code(
rhs, namespace_name=lhs.namespace.value)
super().__init__(WithType, Keyword('do'), Code, None, action)
def __init__(self):
super().__init__(Keyword('while'), Code, WhileType, lambda v, i: v)
def __init__(self):
super().__init__(Keyword('for'), Array, ForSpecType, lambda v, i: v)
def __init__(self):
super().__init__(Code, Keyword('else'), Code, ElseType,
lambda lhs, rhs, i: (lhs, rhs))
def __init__(self, action=None):
super().__init__(ForSpecType, Keyword('do'), Code, None, action)
def test_code(self):
self.assertEqual('{_x=2;}', str(Code([Statement([V('_x'), Keyword('='), N(2)], ending=';')])))
def __init__(self):
super().__init__(Keyword('for'), String, ForType, lambda rhs, i: rhs)
def test_keyword(self):
s = Statement([Space(), Keyword('for')])
s.set_position((1, 1))
self.assertEqual((1, 2), s[1].position)
def __init__(self):
super().__init__(
ForType, Keyword('step'), Number, ForType, lambda lhs, rhs, i:
(lhs.variable, lhs.from_, lhs.to, rhs))
for expression in EXPRESSIONS:
if isinstance(expression, BinaryExpression):
op = expression.types_or_values[1]
BINARY_OPERATORS.add(op.value.lower())
elif isinstance(expression, UnaryExpression):
op = expression.types_or_values[0]
UNARY_OPERATORS.add(op.value.lower())
else:
op = expression.types_or_values[0]
NULARY_OPERATORS.add(op)
KEYWORDS.add(op.value.lower())
OP_ARITHMETIC = [
Keyword(s) for s in ('+', '-', '*', '/', '%', 'mod', '^', 'max', 'floor')
]
OP_LOGICAL = [Keyword(s) for s in ('&&', 'and', '||', 'or')]
OP_COMPARISON = [
Keyword(s) for s in ('==', 'isequalto', '!=', '<', '>', '<=', '>=', '>>')
]
NAMESPACES = {
'missionnamespace', 'profilenamespace', 'uinamespace', 'parsingnamespace'
}
# namespaces are parsed as such
KEYWORDS = KEYWORDS - NAMESPACES
TryCatchExpression(),
ForEachExpression(),
WhileExpression(),
WhileDoExpression(),
ForExpression(),
ForFromExpression(),
ForFromToExpression(),
ForFromToStepExpression(),
ForFromToDoExpression(),
ForSpecExpression(),
ForSpecDoExpression(),
SwitchExpression(),
SwitchDoExpression(),
IfExpression(),
ElseExpression(),
IfThenSpecExpression(),
IfThenElseExpression(),
IfThenExpression(),
IfThenExitWithExpression(),
UnaryExpression(Keyword('params'), Array, Boolean,
lambda rhs_v, i: i.add_params(rhs_v)),
BinaryExpression(Type, Keyword('params'), Array, Boolean,
lambda lhs_v, rhs_v, i: i.add_params(rhs_v, lhs_v)),
UnaryExpression(Keyword('call'), Code, None,
lambda rhs_v, i: i.execute_code(rhs_v)),
BinaryExpression(
Type, Keyword('call'), Code, None,
lambda lhs_v, rhs_v, i: i.execute_code(rhs_v,
extra_scope={"_this": lhs_v})),
]
