def char_literal(char_stream, location):
char = consume(char_stream) and consume(char_stream) # consume initial single quote, consume char
if char == TOKENS.SINGLE_QUOTE: # empty char ...
return CHAR('', location)
if char == '\\': # if char is being escaped
char = escape_characters.get(peek(char_stream), consume(char_stream))
return error_if_not_value(char_stream, TOKENS.SINGLE_QUOTE) and CHAR(char, location)
def _merge_tokens(tokens):
for token in imap(consume, repeat(tokens)):
if token == TOKENS.PP:
token = IGNORE()
while peek_or_terminal(tokens) == TOKENS.PP and consume(
tokens):
new_token_source = token + consume(tokens, IGNORE())
new_tokens = tokenize(
imap(
Str, new_token_source,
imap(
Location,
repeat(
loc(token).file_name,
len(new_token_source)),
repeat(loc(token).line_number),
count(loc(token).column_number),
)))
token = next(new_tokens, IGNORE())
terminal_token = next(new_tokens, terminal)
if terminal_token is not terminal:
raise ValueError(
'{l} token pasting generated more than one token {t} {e}'
.format(l=loc(token), t=token, e=terminal_token))
if token == TOKENS.PP:
token = IGNORE()
yield token
def composite_specifier(
tokens,
symbol_table,
obj_type=StructType,
member_parse_func=parse_struct_members,
terminal=object()
):
"""
: 'composite type' IDENTIFIER
| 'composite type' IDENTIFIER '{' members '}'
| 'composite type' '{' members '}'
"""
location = loc(consume(tokens))
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE: # anonymous composite ...
return obj_type(None, member_parse_func(tokens, symbol_table), location)
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
obj = symbol_table.get(obj_type.get_name(peek(tokens)), obj_type(consume(tokens), None, location))
# some composites are bit tricky such as Struct/Union ...
# since any of its members may contain itself as a reference, so we'll add the type to
# the symbol table before adding the members ...
# TODO: make types immutable, right now they are being shared.
if symbol_table.get(obj.name, terminal) is terminal:
symbol_table[name(obj)] = obj
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
obj.members = member_parse_func(tokens, symbol_table)
return obj
raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
))
def UNDEF(token_seq, macros):
line = get_line(token_seq)
macro_name = consume(line) and error_if_not_type(
consume(line, EOFLocation), (IDENTIFIER, KEYWORD))
_ = macro_name in macros and macros.pop(macro_name)
_ = error_if_not_empty(line)
yield IGNORE(location=loc(macro_name))
def composite_specifier(tokens,
symbol_table,
obj_type=StructType,
member_parse_func=parse_struct_members,
terminal=object()):
"""
: 'composite type' IDENTIFIER
| 'composite type' IDENTIFIER '{' members '}'
| 'composite type' '{' members '}'
"""
location = loc(consume(tokens))
if peek_or_terminal(
tokens) == TOKENS.LEFT_BRACE: # anonymous composite ...
return obj_type(None, member_parse_func(tokens, symbol_table),
location)
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
obj = symbol_table.get(obj_type.get_name(peek(tokens)),
obj_type(consume(tokens), None, location))
# some composites are bit tricky such as Struct/Union ...
# since any of its members may contain itself as a reference, so we'll add the type to
# the symbol table before adding the members ...
# TODO: make types immutable, right now they are being shared.
if symbol_table.get(obj.name, terminal) is terminal:
symbol_table[name(obj)] = obj
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
obj.members = member_parse_func(tokens, symbol_table)
return obj
raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')))
def _values(char_stream):
while peek(char_stream, TOKENS.DOUBLE_QUOTE) != TOKENS.DOUBLE_QUOTE:
value = consume(char_stream)
value = escape_characters.get(
peek(char_stream),
consume(char_stream)) if value == '\\' else value
yield value
_ = error_if_not_value(char_stream, TOKENS.DOUBLE_QUOTE)
def number(char_stream, hexadecimal_chars={'x', 'X'}):
initial_char, _digits = '', digits
if peek_or_terminal(char_stream) == digit(0):
initial_char = consume(char_stream)
if peek_or_terminal(char_stream) in hexadecimal_chars:
initial_char += consume(char_stream)
_digits = hexadecimal_digits
return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))
def arrow_operator(tokens, symbol_table, primary_exp):
l = loc(consume(tokens))
_ = error_if_not_type(c_type(primary_exp), PointerType), \
error_if_not_type(c_type(c_type(primary_exp)), (StructType, UnionType))
member_name = error_if_not_type(consume(tokens, EOFLocation), IDENTIFIER)
return ElementSelectionThroughPointerExpression(
primary_exp, member_name,
c_type(member(c_type(c_type(primary_exp)), member_name))(l), l)
def unary_operator(tokens, symbol_table):
operator = consume(tokens)
if operator == TOKENS.LOGICAL_AND:
return AddressOfLabelExpression(
error_if_not_type(consume(tokens, ''), IDENTIFIER), void_pointer_type(loc(operator)), loc(operator)
)
cast_exp = symbol_table['__ cast_expression __'](tokens, symbol_table)
return rules(unary_operator)[operator](cast_exp, operator)
def number(char_stream, hexadecimal_chars={'x', 'X'}):
initial_char, _digits = '', digits
if peek_or_terminal(char_stream) == digit(0):
initial_char = consume(char_stream)
if peek_or_terminal(char_stream) in hexadecimal_chars:
initial_char += consume(char_stream)
_digits = hexadecimal_digits
return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))
def arrow_operator(tokens, symbol_table, primary_exp):
l = loc(consume(tokens))
_ = error_if_not_type(c_type(primary_exp), PointerType), \
error_if_not_type(c_type(c_type(primary_exp)), (StructType, UnionType))
member_name = error_if_not_type(consume(tokens, EOFLocation), IDENTIFIER)
return ElementSelectionThroughPointerExpression(
primary_exp, member_name, c_type(member(c_type(c_type(primary_exp)), member_name))(l), l
)
def _values(char_stream):
while peek_or_terminal(char_stream) is not terminal:
current_value = consume(char_stream)
yield current_value
# we have consumed a star check if its adjacent value is a forward slash if it is consume and break
if current_value == TOKENS.STAR and peek_or_terminal(char_stream) == TOKENS.FORWARD_SLASH:
yield consume(char_stream)
break
def unary_operator(tokens, symbol_table):
operator = consume(tokens)
if operator == TOKENS.LOGICAL_AND:
return AddressOfLabelExpression(
error_if_not_type(consume(tokens, ''), IDENTIFIER),
void_pointer_type(loc(operator)), loc(operator))
cast_exp = symbol_table['__ cast_expression __'](tokens, symbol_table)
return rules(unary_operator)[operator](cast_exp, operator)
def pre_processor(char_stream, location): # returns pre_processing symbol or #identifier ...
values = consume(char_stream)
if peek_or_terminal(char_stream) == TOKENS.NUMBER_SIGN: # token concatenation symbol ...
values += consume(char_stream)
else:
_ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
values += ''.join(takewhile(letters.__contains__, char_stream))
return rules(pre_processor).get(values, IDENTIFIER)(values, location)
def dot_oper(tokens, symbol_table, primary_exp):
l = (error_if_not_type(c_type(primary_exp),
(StructType, UnionType)) or 1) and loc(
consume(tokens))
member_name = error_if_not_type(consume(tokens, ''), IDENTIFIER)
return ElementSelectionExpression(
primary_exp, member_name,
c_type(member(c_type(primary_exp), member_name))(l), l)
def _values(char_stream):
while peek_or_terminal(char_stream) is not terminal:
current_value = consume(char_stream)
yield current_value
# we have consumed a star check if its adjacent value is a forward slash if it is consume and break
if current_value == TOKENS.STAR and peek_or_terminal(
char_stream) == TOKENS.FORWARD_SLASH:
yield consume(char_stream)
break
def char_literal(char_stream, location):
char = consume(char_stream) and consume(
char_stream) # consume initial single quote, consume char
if char == TOKENS.SINGLE_QUOTE: # empty char ...
return CHAR('', location)
if char == '\\': # if char is being escaped
char = escape_characters.get(peek(char_stream), consume(char_stream))
return error_if_not_value(char_stream, TOKENS.SINGLE_QUOTE) and CHAR(
char, location)
def pre_processor(
char_stream,
location): # returns pre_processing symbol or #identifier ...
values = consume(char_stream)
if peek_or_terminal(
char_stream
) == TOKENS.NUMBER_SIGN: # token concatenation symbol ...
values += consume(char_stream)
else:
_ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
values += ''.join(takewhile(letters.__contains__, char_stream))
return rules(pre_processor).get(values, IDENTIFIER)(values, location)
def body(self, location, arguments=(), macros=()):
if peek_or_terminal(arguments) == TOKENS.LEFT_PARENTHESIS and consume(
arguments):
name = error_if_not_type(consume(arguments, EOFLocation),
(IDENTIFIER, KEYWORD))
_ = error_if_not_value(arguments, TOKENS.RIGHT_PARENTHESIS)
elif isinstance(peek_or_terminal(arguments), (IDENTIFIER, KEYWORD)):
name = consume(arguments)
else:
raise ValueError(
'{l} Expected either LEFT_PARENTHESIS or IDENTIFIER for function macro defined got {g}'
.format(l=location or EOLLocation, g=peek(arguments, '')))
yield INTEGER(str(int(name in macros)), loc(name))
def DEFINE(token_seq, macros):
line = get_line(token_seq)
define_token = consume(line)
name = consume(line)
value = consume(line, default=IGNORE())
if value == TOKENS.LEFT_PARENTHESIS and column_number(name) + len(name) == column_number(value):
macro = _func_macro_definition(name, line)
else: # object macro
macro = ObjectMacro(name, tuple(filter_out_empty_tokens(chain((value,), line))))
_ = name in macros and macros.pop(name) and logger.warning('{0} Redefining macro {1}'.format(loc(name), name))
macros[name] = macro
yield IGNORE(location=loc(define_token))
def init_declarator(tokens,
symbol_table,
base_type=CType(''),
storage_class=None):
# : declarator ('=' assignment_expression or initializer)?
decl = set_core_type(
symbol_table['__ declarator __'](tokens, symbol_table), base_type)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
decl = Definition(name(decl), c_type(decl),
EmptyExpression(c_type(decl)), loc(decl),
storage_class)
symbol_table[name(
decl
)] = decl # we have to add it to the symbol table for things like `int a = a;`
expr = initializer_or_assignment_expression(tokens, symbol_table)
# if declaration is an array type and the expression is of string_type then convert to initializer for parsing
if isinstance(c_type(decl), ArrayType) and isinstance(
c_type(expr), StringType):
expr = Initializer(
enumerate(exp(expr)),
ArrayType(c_type(c_type(expr)), len(c_type(expr)), loc(expr)),
loc(expr))
decl.initialization = parse_initializer(expr, decl) if isinstance(
expr, Initializer) else expr
else:
symbol_table[name(decl)] = decl = Declaration(name(decl), c_type(decl),
loc(decl))
return decl
def default(tokens, symbol_table):
location, _ = loc(consume(tokens)), error_if_not_value(
tokens, TOKENS.COLON)
switch = symbol_table['__ SWITCH STATEMENT __']
switch['default'] = DefaultStatement(
symbol_table['__ statement __'](tokens, symbol_table), location)
yield switch['default']
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(
takewhile(
lambda i: isinstance(i, Allocate) and isinstance(
opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def _func_macro_arguments(line):
symbol_table = SymbolTable()
while peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS:
if peek(line) == TOKENS.ELLIPSIS:
arg = FunctionMacroVariadicArgument(IDENTIFIER('__VA_ARGS__', loc(consume(line))))
else:
arg = FunctionMacroArgument(error_if_not_type(consume(line, EOFLocation), (IDENTIFIER, KEYWORD)))
if peek_or_terminal(line) == TOKENS.ELLIPSIS:
arg = FunctionMacroVariadicArgument(IDENTIFIER(arg, loc(consume(line))))
symbol_table[arg] = arg # check for duplicate argument name
yield arg # if ok add to the rest ...
if isinstance(arg, FunctionMacroVariadicArgument): # if variadic argument break ...
break
# consume expected comma if we don't see a right parenthesis ...
_ = peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS \
and error_if_not_value(line, TOKENS.COMMA, loc(arg))
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(takewhile(lambda i: isinstance(i, Allocate) and isinstance(opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def string_literal(char_stream, location):
def _values(char_stream):
while peek(char_stream, TOKENS.DOUBLE_QUOTE) != TOKENS.DOUBLE_QUOTE:
value = consume(char_stream)
value = escape_characters.get(peek(char_stream), consume(char_stream)) if value == '\\' else value
yield value
_ = error_if_not_value(char_stream, TOKENS.DOUBLE_QUOTE)
return consume(char_stream) and STRING(''.join(_values(char_stream)), location)
def function_call(tokens, symbol_table, primary_exp):
l = loc(consume(tokens))
func_type = error_if_not_type(c_type(c_type(primary_exp)), FunctionType)
# get expression arguments.
expression_argument_list = ArgumentExpressionList(tuple(get_args(tokens, symbol_table, func_type)), l)
return error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS) and FunctionCallExpression(
primary_exp, expression_argument_list, c_type(func_type)(l), l
)
def DEFINE(token_seq, macros):
line = get_line(token_seq)
define_token = consume(line)
name = consume(line)
value = consume(line, default=IGNORE())
if value == TOKENS.LEFT_PARENTHESIS and column_number(name) + len(
name) == column_number(value):
macro = _func_macro_definition(name, line)
else: # object macro
macro = ObjectMacro(
name, tuple(filter_out_empty_tokens(chain((value, ), line))))
_ = name in macros and macros.pop(name) and logger.warning(
'{0} Redefining macro {1}'.format(loc(name), name))
macros[name] = macro
yield IGNORE(location=loc(define_token))
def argument(
token_seq, # a non empty argument terminates with either a comma or right parenthesis ...
takewhile=lambda token_seq: peek(token_seq, TOKENS.COMMA) not in
{TOKENS.COMMA, TOKENS.RIGHT_PARENTHESIS}):
while takewhile(token_seq):
if peek_or_terminal(
token_seq) == TOKENS.LEFT_PARENTHESIS: # nested parenthesis
yield consume(token_seq)
for token in argument(
token_seq, # recursively call argument chaining all the nested parenthesis, until last right is hit
takewhile=lambda token_seq: peek(
token_seq, TOKENS.RIGHT_PARENTHESIS) != TOKENS.
RIGHT_PARENTHESIS):
yield token
yield error_if_not_value(token_seq, TOKENS.RIGHT_PARENTHESIS)
else:
yield consume(token_seq)
def argument_expression_list(tokens, symbol_table): # : assignment_expression (',' assignment_expression)*
assignment_expression = symbol_table['__ assignment_expression __']
return chain(
(assignment_expression(tokens, symbol_table),),
starmap(
assignment_expression,
takewhile(lambda i: peek(i[0]) == TOKENS.COMMA and consume(i[0]), repeat((tokens, symbol_table)))
)
)
def merge_lines(char_seq):
while True:
char = consume(char_seq)
if char == '\\' and isinstance(peek(char_seq), NewLineStr): # if current char is \ followed by end of line seq
_ = exhaust(takewhile(lambda token: isinstance(token, NewLineStr), char_seq))
for char in get_repositioned_line(char_seq, loc(char)):
yield char
else:
yield char
def parameter_type_list(tokens, symbol_table): # : parameter_declaration (',' parameter_declaration)*
return chain(
(parameter_declaration(tokens, symbol_table),),
imap(
parameter_declaration,
takewhile(lambda tokens: peek(tokens) == TOKENS.COMMA and consume(tokens), repeat(tokens)),
repeat(symbol_table)
)
)
def get_repositioned_line(char_seq, location): # get next line ...
while not isinstance(peek(char_seq), NewLineStr):
char = consume(char_seq)
if char == '\\' and isinstance(peek(char_seq), NewLineStr):
_ = exhaust(takewhile(lambda token: isinstance(token, NewLineStr), char_seq))
for char in get_repositioned_line(char_seq, location):
yield char
else:
yield Str(char, location)
def argument_expression_list(
tokens,
symbol_table): # : assignment_expression (',' assignment_expression)*
assignment_expression = symbol_table['__ assignment_expression __']
return chain(
(assignment_expression(tokens, symbol_table), ),
starmap(
assignment_expression,
takewhile(lambda i: peek(i[0]) == TOKENS.COMMA and consume(i[0]),
repeat((tokens, symbol_table)))))
def initializer_list(tokens, symbol_table):
return () if peek(
tokens, TOKENS.RIGHT_BRACE) == TOKENS.RIGHT_BRACE else chain(
(designated_expression_or_expression(tokens, symbol_table), ),
starmap(
designated_expression_or_expression,
takewhile(
lambda i: peek_or_terminal(i[0]) == TOKENS.COMMA and
consume(i[0]) and peek(tokens, TOKENS.RIGHT_BRACE) !=
TOKENS.RIGHT_BRACE, repeat((tokens, symbol_table)))))
def subscript_oper(tokens, symbol_table, primary_exp):
location = error_if_not_type(c_type(primary_exp), PointerType) and loc(
consume(tokens))
expr = symbol_table['__ expression __'](
tokens, symbol_table) # subscript must be of Integral Type.
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACKET) and error_if_not_type(
c_type(expr), IntegralType)
return ArraySubscriptingExpression(primary_exp, expr,
c_type(c_type(primary_exp))(location),
location)
def symbol(char_stream, location):
def _values(char_stream):
value = ''
while value + peek(char_stream) in TOKENS.non_keyword_symbols:
current_value = consume(char_stream)
value += current_value
yield current_value
value = ''.join(_values(char_stream))
next_char = peek_or_terminal(char_stream)
# if value is a single dot check if the next value is a number for possible float or ellipsis ...
if value == TOKENS.DOT and next_char is not terminal:
if next_char in digits: # check for float ...
return FLOAT(value + number(char_stream), location)
if next_char == TOKENS.DOT: # check for ellipsis ...
value += consume(char_stream)
if peek_or_terminal(char_stream) == TOKENS.DOT:
return SYMBOL(value + consume(char_stream), location) # TOKENS.ELLIPSIS
raise_error('{l} Unable to tokenize: `{t}`'.format(l=location, t=TOKENS.DOT + TOKENS.DOT))
return SYMBOL(value, location)
def function_call(tokens, symbol_table, primary_exp):
l = loc(consume(tokens))
func_type = error_if_not_type(c_type(c_type(primary_exp)), FunctionType)
# get expression arguments.
expression_argument_list = ArgumentExpressionList(
tuple(get_args(tokens, symbol_table, func_type)), l)
return error_if_not_value(
tokens, TOKENS.RIGHT_PARENTHESIS) and FunctionCallExpression(
primary_exp, expression_argument_list,
c_type(func_type)(l), l)
def error_if_not_value(value_stream, value, location=LocationNotSet):
try:
error_if_empty(value_stream, location)
except ValueError as _:
raise ValueError('{l} Expected {value} but got nothing'.format(l=location, value=value))
curr = consume(value_stream)
return (
curr != value and raise_error('{l} Expected {value} but got {got}'.format(
l=loc(curr or location), value=value, got=curr))) or curr
def parse_enum_members(tokens, symbol_table):
constant_expression = symbol_table['__ constant_expression __']
location, members, current_value = loc(consume(tokens)), OrderedDict(), 0
while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
ident = error_if_not_type(consume(tokens, ''), IDENTIFIER)
value = ConstantExpression(current_value, IntegerType(location), location)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
value = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(value), IntegerType)
current_value = error_if_not_type(exp(value), (int, long))
symbol_table[ident] = value # Add value to symbol_table
members[ident] = Definition(ident, c_type(value), value, location)
_ = peek_or_terminal(tokens) == TOKENS.COMMA and consume(tokens)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)
return members
def parameter_type_list(
tokens,
symbol_table): # : parameter_declaration (',' parameter_declaration)*
return chain((parameter_declaration(tokens, symbol_table), ),
imap(
parameter_declaration,
takewhile(
lambda tokens: peek(tokens) == TOKENS.COMMA
and consume(tokens), repeat(tokens)),
repeat(symbol_table)))
def get_repositioned_line(char_seq, location): # get next line ...
while not isinstance(peek(char_seq), NewLineStr):
char = consume(char_seq)
if char == '\\' and isinstance(peek(char_seq), NewLineStr):
_ = exhaust(
takewhile(lambda token: isinstance(token, NewLineStr),
char_seq))
for char in get_repositioned_line(char_seq, location):
yield char
else:
yield Str(char, location)
def switch(tokens, symbol_table):
def _pop_symbol_table(symbol_table): # Pop symbol table once we have gone through the whole body ...
_ = pop(symbol_table)
yield EmptyStatement()
expression, statement = imap(symbol_table.__getitem__, ('__ expression __', '__ statement __'))
location, _ = loc(consume(tokens)), error_if_not_value(tokens, TOKENS.LEFT_PARENTHESIS)
expr, _ = expression(tokens, symbol_table), error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS)
symbol_table = push(symbol_table)
symbol_table['__ SWITCH STATEMENT __'] = SymbolTable() # Add dict to track cases, emit error on duplicates.
symbol_table['__ SWITCH EXPRESSION __'] = expr
yield SwitchStatement(expr, chain(statement(tokens, symbol_table), _pop_symbol_table(symbol_table)), location)
def INCLUDE(token_seq, macros):
line = get_line(token_seq)
file_path = consume(line) and get_rule(INCLUDE, peek_or_terminal(line), hash_funcs=(type, identity))(line, macros)
search_paths = (os.getcwd(),)
_ = error_if_not_empty(line)
return chain(
macros['__ preprocess __'](
tokenize(load(file_path, chain(macros['__ include_dirs __'], search_paths))),
macros
),
)
def parse_enum_members(tokens, symbol_table):
constant_expression = symbol_table['__ constant_expression __']
location, members, current_value = loc(consume(tokens)), OrderedDict(), 0
while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
ident = error_if_not_type(consume(tokens, ''), IDENTIFIER)
value = ConstantExpression(current_value, IntegerType(location),
location)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
value = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(value), IntegerType)
current_value = error_if_not_type(exp(value), (int, long))
symbol_table[ident] = value # Add value to symbol_table
members[ident] = Definition(ident, c_type(value), value, location)
_ = peek_or_terminal(tokens) == TOKENS.COMMA and consume(tokens)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)
return members
def init_declarator_list(tokens, symbol_table, base_type=CType(''), storage_class=None):
return chain( # init_declarator (',' init_declarator)*
(init_declarator(tokens, symbol_table, base_type=base_type, storage_class=storage_class),),
starmap(
init_declarator,
takewhile(
lambda i: peek(i[0]) == TOKENS.COMMA and consume(i[0]),
repeat((tokens, symbol_table, base_type, storage_class))
)
)
)
def INCLUDE(token_seq, macros):
line = get_line(token_seq)
file_path = consume(line) and get_rule(
INCLUDE, peek_or_terminal(line), hash_funcs=(type, identity))(line,
macros)
search_paths = (os.getcwd(), )
_ = error_if_not_empty(line)
return chain(
macros['__ preprocess __'](tokenize(
load(file_path, chain(macros['__ include_dirs __'],
search_paths))), macros), )
def initializer_list(tokens, symbol_table):
return () if peek(tokens, TOKENS.RIGHT_BRACE) == TOKENS.RIGHT_BRACE else chain(
(designated_expression_or_expression(tokens, symbol_table),),
starmap(
designated_expression_or_expression,
takewhile(
lambda i: peek_or_terminal(i[0]) == TOKENS.COMMA and consume(i[0])
and peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE,
repeat((tokens, symbol_table))
)
)
)
def string_literal(char_stream, location):
def _values(char_stream):
while peek(char_stream, TOKENS.DOUBLE_QUOTE) != TOKENS.DOUBLE_QUOTE:
value = consume(char_stream)
value = escape_characters.get(
peek(char_stream),
consume(char_stream)) if value == '\\' else value
yield value
_ = error_if_not_value(char_stream, TOKENS.DOUBLE_QUOTE)
return consume(char_stream) and STRING(''.join(_values(char_stream)),
location)
def _func_macro_arguments(line):
symbol_table = SymbolTable()
while peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS:
if peek(line) == TOKENS.ELLIPSIS:
arg = FunctionMacroVariadicArgument(
IDENTIFIER('__VA_ARGS__', loc(consume(line))))
else:
arg = FunctionMacroArgument(
error_if_not_type(consume(line, EOFLocation),
(IDENTIFIER, KEYWORD)))
if peek_or_terminal(line) == TOKENS.ELLIPSIS:
arg = FunctionMacroVariadicArgument(
IDENTIFIER(arg, loc(consume(line))))
symbol_table[arg] = arg # check for duplicate argument name
yield arg # if ok add to the rest ...
if isinstance(arg, FunctionMacroVariadicArgument
): # if variadic argument break ...
break
# consume expected comma if we don't see a right parenthesis ...
_ = peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS \
and error_if_not_value(line, TOKENS.COMMA, loc(arg))
def positioned_chars(char_seq, file_name, initial_line_number=1, initial_column_number=1):
line_numbers, column_numbers = imap(count, (initial_line_number, initial_column_number))
line_number = next(line_numbers)
for ch in imap(peek, repeat(char_seq)):
if linesep.startswith(ch): # check for an end of line sequence ...
possible_end_of_lines_chars = get_newline_if_possible(char_seq) # get possible end of line sequence
str_type = NewLineStr if possible_end_of_lines_chars == linesep else Str
for c in possible_end_of_lines_chars: # emit sequence ...
yield str_type(c, Location(file_name, line_number, next(column_numbers)))
if str_type is NewLineStr: # if sequence is end of line then reset column # and inc line #
column_numbers, line_number = count(initial_column_number), next(line_numbers)
else:
yield Str(consume(char_seq), Location(file_name, line_number, next(column_numbers)))
def dimensions(tokens):
while peek(tokens) == TOKENS.LEFT_BRACKET:
location = loc(consume(tokens))
if peek(tokens) == TOKENS.RIGHT_BRACKET:
size = None
else:
const_exp = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(const_exp), IntegralType)
if exp(const_exp) < 0:
raise ValueError('{l} array size is negative'.format(l=loc(const_exp)))
size = exp(const_exp)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACKET)
yield size, location
def merge_lines(char_seq):
while True:
char = consume(char_seq)
if char == '\\' and isinstance(
peek(char_seq), NewLineStr
): # if current char is \ followed by end of line seq
_ = exhaust(
takewhile(lambda token: isinstance(token, NewLineStr),
char_seq))
for char in get_repositioned_line(char_seq, loc(char)):
yield char
else:
yield char
def number_literal(char_stream, location):
values, sfix = number(char_stream), suffix(char_stream)
_ = sfix not in possible_numeric_suffix and raise_error('{0} Invalid numeric suffix {1}'.format(location, sfix))
if peek_or_terminal(char_stream) == TOKENS.DOT:
return FLOAT(values + consume(char_stream) + number(char_stream), location)
_token_type = INTEGER
if any(imap(values.startswith, octal_prefix)) and values != digit(0):
_token_type = OCTAL
if any(imap(values.startswith, hexadecimal_prefix)):
_token_type = HEXADECIMAL
return _token_type(values, location, sfix)
def get_binary_expression(tokens, symbol_table, l_exp, right_exp_func, exp_type):
operator = consume(tokens)
r_exp = right_exp_func(tokens, symbol_table)
exp_type = max(imap(error_if_not_type, imap(c_type, (l_exp, r_exp)), repeat(exp_type)))
if operator in LOGICAL_OPERATIONS:
exp_type = logical_type
if operator not in supported_operations(c_type(l_exp)):
raise ValueError('{l} ctype {g} does not support {o}'.format(l=loc(l_exp), g=c_type(l_exp), o=operator))
if operator not in supported_operations(c_type(r_exp)):
raise ValueError('{l} ctype {g} does not support {o}'.format(l=loc(r_exp), g=c_type(r_exp), o=operator))
return BinaryExpression(l_exp, operator, r_exp, exp_type(location=loc(operator)), loc(operator))
def _return(tokens, symbol_table):
location = loc(consume(tokens))
ret_type = symbol_table['__ RETURN_TYPE __']
ret_exp = EmptyExpression(VoidType(location), location)
if peek_or_terminal(tokens) != TOKENS.SEMICOLON:
ret_exp = symbol_table['__ expression __'](tokens, symbol_table)
if not isinstance(ret_exp, EmptyExpression) and isinstance(ret_type, VoidType):
raise ValueError('{l} void-function returning a value ...'.format(l=loc(ret_exp)))
if not safe_type_coercion(c_type(ret_exp), ret_type):
raise ValueError('{l} Unable to coerce from {f} to {t}'.format(l=loc(ret_exp), f=c_type(ret_exp), t=ret_type))
return ReturnStatement(ret_exp, location)
def init_declarator(tokens, symbol_table, base_type=CType(''), storage_class=None):
# : declarator ('=' assignment_expression or initializer)?
decl = set_core_type(symbol_table['__ declarator __'](tokens, symbol_table), base_type)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
decl = Definition(name(decl), c_type(decl), EmptyExpression(c_type(decl)), loc(decl), storage_class)
symbol_table[name(decl)] = decl # we have to add it to the symbol table for things like `int a = a;`
expr = initializer_or_assignment_expression(tokens, symbol_table)
# if declaration is an array type and the expression is of string_type then convert to initializer for parsing
if isinstance(c_type(decl), ArrayType) and isinstance(c_type(expr), StringType):
expr = Initializer(
enumerate(exp(expr)), ArrayType(c_type(c_type(expr)), len(c_type(expr)), loc(expr)), loc(expr)
)
decl.initialization = parse_initializer(expr, decl) if isinstance(expr, Initializer) else expr
else:
symbol_table[name(decl)] = decl = Declaration(name(decl), c_type(decl), loc(decl))
return decl
def parse_struct_members(tokens, symbol_table):
declarator = symbol_table['__ declarator __']
location, members = loc(consume(tokens)), OrderedDict()
while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
type_spec = specifier_qualifier_list(tokens, symbol_table)
while peek(tokens, TOKENS.SEMICOLON) != TOKENS.SEMICOLON:
decl = declarator(tokens, symbol_table)
set_core_type(decl, type_spec)
if name(decl) in members:
raise ValueError('{l} Duplicate struct member {name} previous at {at}'.format(
l=loc(decl), name=name(decl), at=loc(members[name(decl)])
))
members[name(decl)] = decl
_ = peek_or_terminal(tokens) != TOKENS.SEMICOLON and error_if_not_value(tokens, TOKENS.COMMA)
_ = error_if_not_value(tokens, TOKENS.SEMICOLON)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)
return members
