def composite_specifier(tokens,
symbol_table,
obj_type=StructType,
member_parse_func=parse_struct_members,
terminal=object()):
"""
: 'composite type' IDENTIFIER
| 'composite type' IDENTIFIER '{' members '}'
| 'composite type' '{' members '}'
"""
location = loc(consume(tokens))
if peek_or_terminal(
tokens) == TOKENS.LEFT_BRACE: # anonymous composite ...
return obj_type(None, member_parse_func(tokens, symbol_table),
location)
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
obj = symbol_table.get(obj_type.get_name(peek(tokens)),
obj_type(consume(tokens), None, location))
# some composites are bit tricky such as Struct/Union ...
# since any of its members may contain itself as a reference, so we'll add the type to
# the symbol table before adding the members ...
# TODO: make types immutable, right now they are being shared.
if symbol_table.get(obj.name, terminal) is terminal:
symbol_table[name(obj)] = obj
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
obj.members = member_parse_func(tokens, symbol_table)
return obj
raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')))
def expand(token, tokens, macros, expanded_macros=None):
expanded_macros = expanded_macros or {}
if token in macros and token not in expanded_macros:
if peek_or_terminal(tokens) != TOKENS.LEFT_PARENTHESIS and isinstance(
macros[token], FunctionMacro):
yield token # don't expand function macros that aren't followed by a parenthesis ...
else:
token_body = macros[token].body(loc(token), tokens,
macros=macros) # get body ...
expanded_macros[
token] = token # mark token so it doesn't get re expanded ...
for t in expand_all(imap(consume, repeat(token_body)), macros,
expanded_macros): # expand body ...
if peek_or_terminal(
token_body
) is terminal: # if this is the last token check for further expansion
for _t in expand(t, tokens, macros, expanded_macros):
yield _t
else:
yield t
_ = expanded_macros.pop(
token
) # expansion complete mark token so it can be re-expanded ...
else:
yield token
def composite_specifier(
tokens,
symbol_table,
obj_type=StructType,
member_parse_func=parse_struct_members,
terminal=object()
):
"""
: 'composite type' IDENTIFIER
| 'composite type' IDENTIFIER '{' members '}'
| 'composite type' '{' members '}'
"""
location = loc(consume(tokens))
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE: # anonymous composite ...
return obj_type(None, member_parse_func(tokens, symbol_table), location)
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
obj = symbol_table.get(obj_type.get_name(peek(tokens)), obj_type(consume(tokens), None, location))
# some composites are bit tricky such as Struct/Union ...
# since any of its members may contain itself as a reference, so we'll add the type to
# the symbol table before adding the members ...
# TODO: make types immutable, right now they are being shared.
if symbol_table.get(obj.name, terminal) is terminal:
symbol_table[name(obj)] = obj
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
obj.members = member_parse_func(tokens, symbol_table)
return obj
raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
))
def number(char_stream, hexadecimal_chars={'x', 'X'}):
initial_char, _digits = '', digits
if peek_or_terminal(char_stream) == digit(0):
initial_char = consume(char_stream)
if peek_or_terminal(char_stream) in hexadecimal_chars:
initial_char += consume(char_stream)
_digits = hexadecimal_digits
return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))
def _values(char_stream):
while peek_or_terminal(char_stream) is not terminal:
current_value = consume(char_stream)
yield current_value
# we have consumed a star check if its adjacent value is a forward slash if it is consume and break
if current_value == TOKENS.STAR and peek_or_terminal(char_stream) == TOKENS.FORWARD_SLASH:
yield consume(char_stream)
break
def number(char_stream, hexadecimal_chars={'x', 'X'}):
initial_char, _digits = '', digits
if peek_or_terminal(char_stream) == digit(0):
initial_char = consume(char_stream)
if peek_or_terminal(char_stream) in hexadecimal_chars:
initial_char += consume(char_stream)
_digits = hexadecimal_digits
return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))
def _values(char_stream):
while peek_or_terminal(char_stream) is not terminal:
current_value = consume(char_stream)
yield current_value
# we have consumed a star check if its adjacent value is a forward slash if it is consume and break
if current_value == TOKENS.STAR and peek_or_terminal(
char_stream) == TOKENS.FORWARD_SLASH:
yield consume(char_stream)
break
def pointer_type_abstract_declarator(tokens, symbol_table):
decl = AbstractDeclarator(pointer(tokens, symbol_table),
loc(peek_or_terminal(tokens)))
if get_rule(direct_abstract_declarator,
peek_or_terminal(tokens),
None,
hash_funcs=(type, identity)) is not None:
decl = set_core_type(direct_abstract_declarator(tokens, symbol_table),
c_type(decl))
return decl
def type_name(tokens, symbol_table): #: type_specifier abstract_declarator? # returns CType
base_type = specifier_qualifier_list(tokens, symbol_table)
abstract_declarator = symbol_table['__ abstract_declarator __']
if peek_or_terminal(tokens) in {TOKENS.LEFT_PARENTHESIS, TOKENS.LEFT_BRACKET, TOKENS.STAR} \
or isinstance(peek_or_terminal(tokens), IDENTIFIER):
abs_decl = abstract_declarator(tokens, symbol_table)
set_core_type(abs_decl, base_type)
return c_type(abs_decl)
return base_type
def type_name(tokens, symbol_table
): #: type_specifier abstract_declarator? # returns CType
base_type = specifier_qualifier_list(tokens, symbol_table)
abstract_declarator = symbol_table['__ abstract_declarator __']
if peek_or_terminal(tokens) in {TOKENS.LEFT_PARENTHESIS, TOKENS.LEFT_BRACKET, TOKENS.STAR} \
or isinstance(peek_or_terminal(tokens), IDENTIFIER):
abs_decl = abstract_declarator(tokens, symbol_table)
set_core_type(abs_decl, base_type)
return c_type(abs_decl)
return base_type
def body(self, location, arguments=(), macros=()):
if peek_or_terminal(arguments) == TOKENS.LEFT_PARENTHESIS and consume(
arguments):
name = error_if_not_type(consume(arguments, EOFLocation),
(IDENTIFIER, KEYWORD))
_ = error_if_not_value(arguments, TOKENS.RIGHT_PARENTHESIS)
elif isinstance(peek_or_terminal(arguments), (IDENTIFIER, KEYWORD)):
name = consume(arguments)
else:
raise ValueError(
'{l} Expected either LEFT_PARENTHESIS or IDENTIFIER for function macro defined got {g}'
.format(l=location or EOLLocation, g=peek(arguments, '')))
yield INTEGER(str(int(name in macros)), loc(name))
def direct_declarator(tokens, symbol_table):
"""
: (IDENTIFIER | '(' declarator ')') declarator_suffix*
declarator_suffix
: '[' constant_expression ']'
| '[' ']'
| '(' parameter_type_list ')'
| '(' ')'
"""
dec = get_rule(direct_declarator, peek_or_terminal(tokens), hash_funcs=(type, identity))(tokens, symbol_table)
_ = peek_or_terminal(tokens) in rules(declarator_suffix) and set_core_type(
dec, declarator_suffix(tokens, symbol_table))
return dec
def declarations(tokens, symbol_table):
# storage_class_specifier? type_name? init_declarator_list (';' or compound_statement) # declaration
storage_class_specifier, specifier_qualifier_list, statement = imap(
symbol_table.__getitem__,
('__ storage_class_specifier __', '__ specifier_qualifier_list __',
'__ statement __'))
storage_class = storage_class_specifier(tokens, symbol_table)
base_type = specifier_qualifier_list(tokens, symbol_table)
expecting_token = TOKENS.SEMICOLON
if peek_or_terminal(tokens) == TOKENS.SEMICOLON:
yield EmptyDeclaration(loc(consume(tokens)), storage_class)
elif peek_or_terminal(tokens) is terminal:
raise_error(
'{l} Expected TOKENS.COMMA TOKENS.EQUAL TOKENS.SEMICOLON TOKENS.LEFT_BRACE got `{got}`'
.format(l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')))
else:
for dec in init_declarator_list(tokens,
symbol_table,
base_type=base_type,
storage_class=storage_class):
dec.storage_class = storage_class
if isinstance(
storage_class, TypeDef
): # init_declarator_list adds the symbol as a decl to symbol_table
symbol_table[name(dec)] = (symbol_table.pop(
name(dec)) or 1) and c_type(dec) # replace dec by ctype
elif peek_or_terminal(
tokens) == TOKENS.LEFT_BRACE and not error_if_not_type(
c_type(dec), FunctionType):
symbol_table = push(symbol_table)
symbol_table.update(
chain(
imap(
lambda a: (
name(a), a
), # add non variable list parameters to the symbol table ...
ifilterfalse(
lambda c: isinstance(c_type(c), VAListType),
c_type(dec))),
(('__ RETURN_TYPE __', c_type(c_type(dec))),
('__ LABELS __', SymbolTable()))))
yield FunctionDefinition(dec,
next(statement(tokens, symbol_table)))
expecting_token = (pop(symbol_table) or 1) and ''
else:
yield dec
expecting_token = TOKENS.SEMICOLON
_ = expecting_token and error_if_not_value(tokens, expecting_token)
def postfix_expression(tokens, symbol_table):
"""
: primary_expression
( '[' expression ']'
| '(' ')'
| '(' argument_expression_list ')'
| '.' IDENTIFIER
| '->' IDENTIFIER
| '++'
| '--' )*
"""
type_name, expression, initializer, primary_expression = imap(
symbol_table.__getitem__,
('__ type_name __', '__ expression __', '__ initializer __', '__ primary_expression __')
)
# if primary_exp is None:
# if peek_or_terminal(tokens) == TOKENS.LEFT_PARENTHESIS and consume(tokens):
# # Again slight ambiguity since primary_expression may start with '(' expression ')'
# # can't call cast_expression since it will try to call postfix_expression.
# if is_type_name(peek_or_terminal(tokens), symbol_table):
# ctype, _ = type_name(tokens, symbol_table), error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS)
# primary_exp = CompoundLiteral(initializer(tokens, symbol_table), ctype, loc(ctype))
# else: # if we saw a parenthesis and it wasn't a type_name then it must be primary_expr `(` expression `)`
# primary_exp, _ = expression(tokens, symbol_table), error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS)
# else:
# primary_exp = primary_expression(tokens, symbol_table)
primary_exp = primary_expression(tokens, symbol_table)
while peek_or_terminal(tokens) in rules(postfix_expression):
primary_exp = rules(postfix_expression)[peek(tokens)](tokens, symbol_table, primary_exp)
return primary_exp
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(takewhile(lambda i: isinstance(i, Allocate) and isinstance(opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def get_line(
values
): # get all the tokens on the current line, being that preprocessor works on a line-by-line basis
return takewhile(
lambda token, initial_line_number=line_number(peek(
values)): initial_line_number == line_number(token),
values) if peek_or_terminal(values) is not terminal else iter(())
def remove_pass(instrs):
""" replace 1 or more sequences of Pass by the next non-Pass instruction or Pass instruction """
pass_instrs = tuple(takewhile(lambda instr: isinstance(instr, Pass), instrs))
if peek_or_terminal(instrs) is terminal:
yield replace_instrs(pass_instrs[-1], pass_instrs[:-1])
else:
replace_instrs(peek(instrs), pass_instrs)
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(
takewhile(
lambda i: isinstance(i, Allocate) and isinstance(
opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def direct_declarator(tokens, symbol_table):
"""
: (IDENTIFIER | '(' declarator ')') declarator_suffix*
declarator_suffix
: '[' constant_expression ']'
| '[' ']'
| '(' parameter_type_list ')'
| '(' ')'
"""
dec = get_rule(direct_declarator,
peek_or_terminal(tokens),
hash_funcs=(type, identity))(tokens, symbol_table)
_ = peek_or_terminal(tokens) in rules(declarator_suffix) and set_core_type(
dec, declarator_suffix(tokens, symbol_table))
return dec
def init_declarator(tokens,
symbol_table,
base_type=CType(''),
storage_class=None):
# : declarator ('=' assignment_expression or initializer)?
decl = set_core_type(
symbol_table['__ declarator __'](tokens, symbol_table), base_type)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
decl = Definition(name(decl), c_type(decl),
EmptyExpression(c_type(decl)), loc(decl),
storage_class)
symbol_table[name(
decl
)] = decl # we have to add it to the symbol table for things like `int a = a;`
expr = initializer_or_assignment_expression(tokens, symbol_table)
# if declaration is an array type and the expression is of string_type then convert to initializer for parsing
if isinstance(c_type(decl), ArrayType) and isinstance(
c_type(expr), StringType):
expr = Initializer(
enumerate(exp(expr)),
ArrayType(c_type(c_type(expr)), len(c_type(expr)), loc(expr)),
loc(expr))
decl.initialization = parse_initializer(expr, decl) if isinstance(
expr, Initializer) else expr
else:
symbol_table[name(decl)] = decl = Declaration(name(decl), c_type(decl),
loc(decl))
return decl
def _merge_tokens(tokens):
for token in imap(consume, repeat(tokens)):
if token == TOKENS.PP:
token = IGNORE()
while peek_or_terminal(tokens) == TOKENS.PP and consume(
tokens):
new_token_source = token + consume(tokens, IGNORE())
new_tokens = tokenize(
imap(
Str, new_token_source,
imap(
Location,
repeat(
loc(token).file_name,
len(new_token_source)),
repeat(loc(token).line_number),
count(loc(token).column_number),
)))
token = next(new_tokens, IGNORE())
terminal_token = next(new_tokens, terminal)
if terminal_token is not terminal:
raise ValueError(
'{l} token pasting generated more than one token {t} {e}'
.format(l=loc(token), t=token, e=terminal_token))
if token == TOKENS.PP:
token = IGNORE()
yield token
def get_args(tokens, symbol_table, func_type):
if peek_or_terminal(
tokens) != TOKENS.RIGHT_PARENTHESIS: # check the arguments.
for arg_decl, arg in takewhile(
lambda args: not (isinstance(c_type(args[0]), VAListType) and
args[1] is None),
izip_longest(arguments(func_type),
argument_expression_list(tokens, symbol_table))):
if arg is None:
raise ValueError(
'{l} Function call with not enough arguments specified.'.
format(l=l))
elif arg_decl is None:
raise ValueError(
'{l} Function call with to many arguments specified'.
format(l=loc(arg)))
elif not safe_type_coercion(c_type(arg), c_type(arg_decl)):
raise ValueError(
'{l} Function call, could not coerce argument from {f_type} to {t_type}'
.format(
l=loc(arg),
f_type=c_type(arg),
t_type=c_type(arg_decl),
))
yield CastExpression(
arg,
c_type(arg if isinstance(c_type(arg_decl), VAListType
) else arg_decl), loc(arg))
def postfix_expression(tokens, symbol_table):
"""
: primary_expression
( '[' expression ']'
| '(' ')'
| '(' argument_expression_list ')'
| '.' IDENTIFIER
| '->' IDENTIFIER
| '++'
| '--' )*
"""
type_name, expression, initializer, primary_expression = imap(
symbol_table.__getitem__,
('__ type_name __', '__ expression __', '__ initializer __',
'__ primary_expression __'))
# if primary_exp is None:
# if peek_or_terminal(tokens) == TOKENS.LEFT_PARENTHESIS and consume(tokens):
# # Again slight ambiguity since primary_expression may start with '(' expression ')'
# # can't call cast_expression since it will try to call postfix_expression.
# if is_type_name(peek_or_terminal(tokens), symbol_table):
# ctype, _ = type_name(tokens, symbol_table), error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS)
# primary_exp = CompoundLiteral(initializer(tokens, symbol_table), ctype, loc(ctype))
# else: # if we saw a parenthesis and it wasn't a type_name then it must be primary_expr `(` expression `)`
# primary_exp, _ = expression(tokens, symbol_table), error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS)
# else:
# primary_exp = primary_expression(tokens, symbol_table)
primary_exp = primary_expression(tokens, symbol_table)
while peek_or_terminal(tokens) in rules(postfix_expression):
primary_exp = rules(postfix_expression)[peek(tokens)](tokens,
symbol_table,
primary_exp)
return primary_exp
def remove_pass(instrs):
""" replace 1 or more sequences of Pass by the next non-Pass instruction or Pass instruction """
pass_instrs = tuple(
takewhile(lambda instr: isinstance(instr, Pass), instrs))
if peek_or_terminal(instrs) is terminal:
yield replace_instrs(pass_instrs[-1], pass_instrs[:-1])
else:
replace_instrs(peek(instrs), pass_instrs)
def expression(
tokens,
symbol_table): # assignment_expression (',' assignment_expression)*
assignment_expression = symbol_table['__ assignment_expression __']
expr = assignment_expression(tokens, symbol_table)
if peek_or_terminal(tokens) == TOKENS.COMMA:
exprs = tuple(
chain((expr, ),
starmap(
assignment_expression,
takewhile(
lambda a: peek_or_terminal(a[0]) == TOKENS.COMMA and
consume(a[0]), repeat((tokens, symbol_table))))))
expr = CommaExpression(exprs, c_type(exprs[-1]), loc(exprs[-1]))
return expr
def pre_processor(char_stream, location): # returns pre_processing symbol or #identifier ...
values = consume(char_stream)
if peek_or_terminal(char_stream) == TOKENS.NUMBER_SIGN: # token concatenation symbol ...
values += consume(char_stream)
else:
_ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
values += ''.join(takewhile(letters.__contains__, char_stream))
return rules(pre_processor).get(values, IDENTIFIER)(values, location)
def expression(tokens, symbol_table): # assignment_expression (',' assignment_expression)*
assignment_expression = symbol_table['__ assignment_expression __']
expr = assignment_expression(tokens, symbol_table)
if peek_or_terminal(tokens) == TOKENS.COMMA:
exprs = tuple(chain(
(expr,),
starmap(
assignment_expression,
takewhile(
lambda a: peek_or_terminal(a[0]) == TOKENS.COMMA and consume(a[0]),
repeat((tokens, symbol_table))
)
)
))
expr = CommaExpression(exprs, c_type(exprs[-1]), loc(exprs[-1]))
return expr
def is_declaration(tokens, symbol_table):
funcs = '__ storage_class_specifier __', '__ type_specifier __', '__ type_qualifiers __'
val = peek_or_terminal(tokens)
return any(chain(
imap(
apply, imap(getattr, imap(rules, get_symbols(symbol_table, *funcs)), repeat('__contains__')), repeat((val,))
),
(isinstance(symbol_table.get(val, val), CType),)
))
def parse_enum_members(tokens, symbol_table):
constant_expression = symbol_table['__ constant_expression __']
location, members, current_value = loc(consume(tokens)), OrderedDict(), 0
while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
ident = error_if_not_type(consume(tokens, ''), IDENTIFIER)
value = ConstantExpression(current_value, IntegerType(location), location)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
value = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(value), IntegerType)
current_value = error_if_not_type(exp(value), (int, long))
symbol_table[ident] = value # Add value to symbol_table
members[ident] = Definition(ident, c_type(value), value, location)
_ = peek_or_terminal(tokens) == TOKENS.COMMA and consume(tokens)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)
return members
def symbol(char_stream, location):
def _values(char_stream):
value = ''
while value + peek(char_stream) in TOKENS.non_keyword_symbols:
current_value = consume(char_stream)
value += current_value
yield current_value
value = ''.join(_values(char_stream))
next_char = peek_or_terminal(char_stream)
# if value is a single dot check if the next value is a number for possible float or ellipsis ...
if value == TOKENS.DOT and next_char is not terminal:
if next_char in digits: # check for float ...
return FLOAT(value + number(char_stream), location)
if next_char == TOKENS.DOT: # check for ellipsis ...
value += consume(char_stream)
if peek_or_terminal(char_stream) == TOKENS.DOT:
return SYMBOL(value + consume(char_stream), location) # TOKENS.ELLIPSIS
raise_error('{l} Unable to tokenize: `{t}`'.format(l=location, t=TOKENS.DOT + TOKENS.DOT))
return SYMBOL(value, location)
def default_parameter_declaration(tokens, symbol_table):
base_type = symbol_table['__ specifier_qualifier_list __'](tokens, symbol_table)
c_decl = AbstractDeclarator(base_type, loc(base_type))
token = peek_or_terminal(tokens)
if token in {TOKENS.STAR, TOKENS.LEFT_PARENTHESIS, TOKENS.LEFT_BRACKET} or isinstance(token, IDENTIFIER):
c_decl = abstract_declarator(tokens, symbol_table)
set_core_type(c_decl, base_type)
return c_decl
def is_declaration(tokens, symbol_table):
funcs = '__ storage_class_specifier __', '__ type_specifier __', '__ type_qualifiers __'
val = peek_or_terminal(tokens)
return any(
chain(
imap(
apply,
imap(getattr, imap(rules, get_symbols(symbol_table, *funcs)),
repeat('__contains__')), repeat((val, ))),
(isinstance(symbol_table.get(val, val), CType), )))
def statement(tokens, symbol_table):
"""
: declaration
| labeled_statement
| compound_statement
| selection_statement
| iteration_statement
| jump_statement
| expression_statement
| expression ';'
| ;
"""
if peek_or_terminal(tokens) in rules(
statement): # if current token has a rule use that one first
return rules(statement)[peek(tokens)](tokens, symbol_table)
if is_declaration(
tokens,
symbol_table): # checking for type_name is a bit expensive ...
return declaration(tokens, symbol_table)
# both expressions and labels may start with an identifier
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
label_name = consume(tokens)
if peek_or_terminal(tokens) == TOKENS.COLON:
return symbol_table['__ labeled_statement __'](chain(
(label_name, ), consume_all(tokens)), symbol_table)
# return label_stmnt(label_name, statement(tokens, symbol_table))
# it must be an expression, TODO: figure out a way without using dangerous chain!
# tokens = chain((label_name, consume(tokens)), tokens)
tokens = chain((label_name, ), consume_all(tokens))
expr, _ = symbol_table['__ expression __'](
tokens, symbol_table), error_if_not_value(tokens, TOKENS.SEMICOLON)
return repeat(expr, 1)
if peek_or_terminal(tokens) is not terminal:
expr, _ = symbol_table['__ expression __'](
tokens, symbol_table), error_if_not_value(tokens, TOKENS.SEMICOLON)
return repeat(expr, 1)
raise ValueError(
'{l} No rule could be found to create statement, got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')))
def offset_designated_expr(tokens, symbol_table): # '[' positive_integral (... positive_integral)? ']'
constant_expression = error_if_not_value(tokens, TOKENS.LEFT_BRACKET) and symbol_table['__ constant_expression __']
designation = NumericalDesignation(exp(constant_expression(tokens, symbol_table)))
if peek_or_terminal(tokens) == TOKENS.ELLIPSIS:
return range_designated_expr(designation, tokens, symbol_table)
return error_if_not_value(tokens, TOKENS.RIGHT_BRACKET) and OffsetDesignatedExpression(
designation, _expr_or_designated_expr(tokens, symbol_table)
)
def is_value_adjacent(values, current_location, value):
next_value = peek_or_terminal(values)
next_location = loc(next_value, EOFLocation)
return not any(
starmap(
cmp,
izip((value, current_location.line_number,
current_location.column_number + len(value)),
(next_value, next_location.line_number,
next_location.column_number))))
def initializer_list(tokens, symbol_table):
return () if peek(
tokens, TOKENS.RIGHT_BRACE) == TOKENS.RIGHT_BRACE else chain(
(designated_expression_or_expression(tokens, symbol_table), ),
starmap(
designated_expression_or_expression,
takewhile(
lambda i: peek_or_terminal(i[0]) == TOKENS.COMMA and
consume(i[0]) and peek(tokens, TOKENS.RIGHT_BRACE) !=
TOKENS.RIGHT_BRACE, repeat((tokens, symbol_table)))))
def INCLUDE(token_seq, macros):
line = get_line(token_seq)
file_path = consume(line) and get_rule(
INCLUDE, peek_or_terminal(line), hash_funcs=(type, identity))(line,
macros)
search_paths = (os.getcwd(), )
_ = error_if_not_empty(line)
return chain(
macros['__ preprocess __'](tokenize(
load(file_path, chain(macros['__ include_dirs __'],
search_paths))), macros), )
def type_name_or_postfix_expression(tokens, symbol_table):
symbol_table = push(symbol_table)
symbol_table['__ compound_literal __'] = type_name_or_compound_literal
primary_exp = symbol_table['__ primary_expression __'](tokens, symbol_table)
_ = pop(symbol_table)
# pop 'type_name_or_compound_literal' and type_name_or_postfix_expression ...
postfix_expression_rules = rules(symbol_table['__ postfix_expression __'])
if not isinstance(primary_exp, CType): # it must have being an expression ...
while peek_or_terminal(tokens) in postfix_expression_rules:
primary_exp = postfix_expression_rules[peek(tokens)](tokens, symbol_table, primary_exp)
return primary_exp # otherwise it must have being a type_name ...
def parse_enum_members(tokens, symbol_table):
constant_expression = symbol_table['__ constant_expression __']
location, members, current_value = loc(consume(tokens)), OrderedDict(), 0
while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
ident = error_if_not_type(consume(tokens, ''), IDENTIFIER)
value = ConstantExpression(current_value, IntegerType(location),
location)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
value = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(value), IntegerType)
current_value = error_if_not_type(exp(value), (int, long))
symbol_table[ident] = value # Add value to symbol_table
members[ident] = Definition(ident, c_type(value), value, location)
_ = peek_or_terminal(tokens) == TOKENS.COMMA and consume(tokens)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)
return members
def for_stmnt(tokens, symbol_table):
location, _ = loc(error_if_not_value(tokens, TOKENS.FOR)), error_if_not_value(tokens, TOKENS.LEFT_PARENTHESIS)
statement, expression = symbol_table['__ statement __'], symbol_table['__ expression __']
init_exp = EmptyExpression(VoidType(location), location)
if peek_or_terminal(tokens) != TOKENS.SEMICOLON:
init_exp = expression(tokens, symbol_table)
_ = error_if_not_value(tokens, TOKENS.SEMICOLON)
conditional_exp = TrueExpression(location)
if peek_or_terminal(tokens) != TOKENS.SEMICOLON:
conditional_exp = expression(tokens, symbol_table)
_ = error_if_not_value(tokens, TOKENS.SEMICOLON)
update_exp = EmptyExpression(VoidType(location), location)
if peek_or_terminal(tokens) != TOKENS.RIGHT_PARENTHESIS:
update_exp = expression(tokens, symbol_table)
_ = error_if_not_value(tokens, TOKENS.RIGHT_PARENTHESIS)
yield ForStatement(init_exp, conditional_exp, update_exp, statement(tokens, symbol_table), location)
def INCLUDE(token_seq, macros):
line = get_line(token_seq)
file_path = consume(line) and get_rule(INCLUDE, peek_or_terminal(line), hash_funcs=(type, identity))(line, macros)
search_paths = (os.getcwd(),)
_ = error_if_not_empty(line)
return chain(
macros['__ preprocess __'](
tokenize(load(file_path, chain(macros['__ include_dirs __'], search_paths))),
macros
),
)
def is_value_adjacent(values, current_location, value):
next_value = peek_or_terminal(values)
next_location = loc(next_value, EOFLocation)
return not any(
starmap(
cmp,
izip(
(value, current_location.line_number, current_location.column_number + len(value)),
(next_value, next_location.line_number, next_location.column_number)
)
)
)
def pre_processor(
char_stream,
location): # returns pre_processing symbol or #identifier ...
values = consume(char_stream)
if peek_or_terminal(
char_stream
) == TOKENS.NUMBER_SIGN: # token concatenation symbol ...
values += consume(char_stream)
else:
_ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
values += ''.join(takewhile(letters.__contains__, char_stream))
return rules(pre_processor).get(values, IDENTIFIER)(values, location)
def initializer_list(tokens, symbol_table):
return () if peek(tokens, TOKENS.RIGHT_BRACE) == TOKENS.RIGHT_BRACE else chain(
(designated_expression_or_expression(tokens, symbol_table),),
starmap(
designated_expression_or_expression,
takewhile(
lambda i: peek_or_terminal(i[0]) == TOKENS.COMMA and consume(i[0])
and peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE,
repeat((tokens, symbol_table))
)
)
)
def default_parameter_declaration(tokens, symbol_table):
base_type = symbol_table['__ specifier_qualifier_list __'](tokens,
symbol_table)
c_decl = AbstractDeclarator(base_type, loc(base_type))
token = peek_or_terminal(tokens)
if token in {TOKENS.STAR, TOKENS.LEFT_PARENTHESIS, TOKENS.LEFT_BRACKET
} or isinstance(token, IDENTIFIER):
c_decl = abstract_declarator(tokens, symbol_table)
set_core_type(c_decl, base_type)
return c_decl
def number_literal(char_stream, location):
values, sfix = number(char_stream), suffix(char_stream)
_ = sfix not in possible_numeric_suffix and raise_error('{0} Invalid numeric suffix {1}'.format(location, sfix))
if peek_or_terminal(char_stream) == TOKENS.DOT:
return FLOAT(values + consume(char_stream) + number(char_stream), location)
_token_type = INTEGER
if any(imap(values.startswith, octal_prefix)) and values != digit(0):
_token_type = OCTAL
if any(imap(values.startswith, hexadecimal_prefix)):
_token_type = HEXADECIMAL
return _token_type(values, location, sfix)
def symbol(char_stream, location):
def _values(char_stream):
value = ''
while value + peek(char_stream) in TOKENS.non_keyword_symbols:
current_value = consume(char_stream)
value += current_value
yield current_value
value = ''.join(_values(char_stream))
next_char = peek_or_terminal(char_stream)
# if value is a single dot check if the next value is a number for possible float or ellipsis ...
if value == TOKENS.DOT and next_char is not terminal:
if next_char in digits: # check for float ...
return FLOAT(value + number(char_stream), location)
if next_char == TOKENS.DOT: # check for ellipsis ...
value += consume(char_stream)
if peek_or_terminal(char_stream) == TOKENS.DOT:
return SYMBOL(value + consume(char_stream),
location) # TOKENS.ELLIPSIS
raise_error('{l} Unable to tokenize: `{t}`'.format(l=location,
t=TOKENS.DOT +
TOKENS.DOT))
return SYMBOL(value, location)
def unary_expression(tokens, symbol_table):
"""
: postfix_expression
| '++' unary_expression
| '--' unary_expression
| unary_operator cast_expression
| 'sizeof' (type_name | unary_expression)
"""
error_if_empty(tokens)
if peek_or_terminal(tokens) in rules(unary_expression) and not isinstance(peek(tokens), CONSTANT):
return rules(unary_expression)[peek(tokens)](tokens, symbol_table)
return symbol_table['__ postfix_expression __'](tokens, symbol_table)
def statement(tokens, symbol_table):
"""
: declaration
| labeled_statement
| compound_statement
| selection_statement
| iteration_statement
| jump_statement
| expression_statement
| expression ';'
| ;
"""
if peek_or_terminal(tokens) in rules(statement): # if current token has a rule use that one first
return rules(statement)[peek(tokens)](tokens, symbol_table)
if is_declaration(tokens, symbol_table): # checking for type_name is a bit expensive ...
return declaration(tokens, symbol_table)
# both expressions and labels may start with an identifier
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
label_name = consume(tokens)
if peek_or_terminal(tokens) == TOKENS.COLON:
return symbol_table['__ labeled_statement __'](chain((label_name,), consume_all(tokens)), symbol_table)
# return label_stmnt(label_name, statement(tokens, symbol_table))
# it must be an expression, TODO: figure out a way without using dangerous chain!
# tokens = chain((label_name, consume(tokens)), tokens)
tokens = chain((label_name,), consume_all(tokens))
expr, _ = symbol_table['__ expression __'](tokens, symbol_table), error_if_not_value(tokens, TOKENS.SEMICOLON)
return repeat(expr, 1)
if peek_or_terminal(tokens) is not terminal:
expr, _ = symbol_table['__ expression __'](tokens, symbol_table), error_if_not_value(tokens, TOKENS.SEMICOLON)
return repeat(expr, 1)
raise ValueError('{l} No rule could be found to create statement, got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
))
def number_literal(char_stream, location):
values, sfix = number(char_stream), suffix(char_stream)
_ = sfix not in possible_numeric_suffix and raise_error(
'{0} Invalid numeric suffix {1}'.format(location, sfix))
if peek_or_terminal(char_stream) == TOKENS.DOT:
return FLOAT(values + consume(char_stream) + number(char_stream),
location)
_token_type = INTEGER
if any(imap(values.startswith, octal_prefix)) and values != digit(0):
_token_type = OCTAL
if any(imap(values.startswith, hexadecimal_prefix)):
_token_type = HEXADECIMAL
return _token_type(values, location, sfix)
def _return(tokens, symbol_table):
location = loc(consume(tokens))
ret_type = symbol_table['__ RETURN_TYPE __']
ret_exp = EmptyExpression(VoidType(location), location)
if peek_or_terminal(tokens) != TOKENS.SEMICOLON:
ret_exp = symbol_table['__ expression __'](tokens, symbol_table)
if not isinstance(ret_exp, EmptyExpression) and isinstance(ret_type, VoidType):
raise ValueError('{l} void-function returning a value ...'.format(l=loc(ret_exp)))
if not safe_type_coercion(c_type(ret_exp), ret_type):
raise ValueError('{l} Unable to coerce from {f} to {t}'.format(l=loc(ret_exp), f=c_type(ret_exp), t=ret_type))
return ReturnStatement(ret_exp, location)
def declarations(tokens, symbol_table):
# storage_class_specifier? type_name? init_declarator_list (';' or compound_statement) # declaration
storage_class_specifier, specifier_qualifier_list, statement = imap(
symbol_table.__getitem__,
('__ storage_class_specifier __', '__ specifier_qualifier_list __', '__ statement __')
)
storage_class = storage_class_specifier(tokens, symbol_table)
base_type = specifier_qualifier_list(tokens, symbol_table)
expecting_token = TOKENS.SEMICOLON
if peek_or_terminal(tokens) == TOKENS.SEMICOLON:
yield EmptyDeclaration(loc(consume(tokens)), storage_class)
elif peek_or_terminal(tokens) is terminal:
raise_error('{l} Expected TOKENS.COMMA TOKENS.EQUAL TOKENS.SEMICOLON TOKENS.LEFT_BRACE got `{got}`'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
))
else:
for dec in init_declarator_list(tokens, symbol_table, base_type=base_type, storage_class=storage_class):
dec.storage_class = storage_class
if isinstance(storage_class, TypeDef): # init_declarator_list adds the symbol as a decl to symbol_table
symbol_table[name(dec)] = (symbol_table.pop(name(dec)) or 1) and c_type(dec) # replace dec by ctype
elif peek_or_terminal(tokens) == TOKENS.LEFT_BRACE and not error_if_not_type(c_type(dec), FunctionType):
symbol_table = push(symbol_table)
symbol_table.update(chain(
imap(
lambda a: (name(a), a), # add non variable list parameters to the symbol table ...
ifilterfalse(lambda c: isinstance(c_type(c), VAListType), c_type(dec))
),
(('__ RETURN_TYPE __', c_type(c_type(dec))), ('__ LABELS __', SymbolTable()))
))
yield FunctionDefinition(dec, next(statement(tokens, symbol_table)))
expecting_token = (pop(symbol_table) or 1) and ''
else:
yield dec
expecting_token = TOKENS.SEMICOLON
_ = expecting_token and error_if_not_value(tokens, expecting_token)
def arguments(token_seq, parameters, l=LocationNotSet):
parameters = iter(parameters)
# empty (no) arguments ... but expects at least one parameter ... so use empty string ...
if peek(token_seq, TOKENS.RIGHT_PARENTHESIS) == TOKENS.RIGHT_PARENTHESIS \
and consume(parameters, terminal) is not terminal:
yield IGNORE(location=(loc(peek(token_seq)) or l)),
while peek(token_seq,
TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS:
if isinstance(peek_or_terminal(parameters),
FunctionMacroVariadicArgument):
tokens = (IGNORE(location=loc(peek(token_seq))),) \
if peek(token_seq) == TOKENS.RIGHT_PARENTHESIS else argument(
token_seq, # if the current parameter is variadic argument get everything including commas ...
takewhile=lambda token_seq: peek(token_seq, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS
) # if at the end of arguments emit emtpy string ...
elif peek_or_terminal(
token_seq
) == TOKENS.COMMA: # if comma than argument is just an empty string ...
tokens = IGNORE(location=loc(peek(token_seq))),
else:
tokens = argument(token_seq)
_ = consume(parameters, None)
yield tokens
if peek_or_terminal(
token_seq
) != TOKENS.RIGHT_PARENTHESIS: # if not at end we are expecting a comma ...
location = loc(error_if_not_value(token_seq, TOKENS.COMMA, l))
if peek_or_terminal(token_seq) == TOKENS.RIGHT_PARENTHESIS \
and isinstance(peek_or_terminal(parameters), FunctionMacroArgument):
_ = consume(
parameters
) # if we read a comma and we are at end still expect at least one more parameter
yield IGNORE(location=location), # yield empty string ...
if isinstance(consume(parameters, None), FunctionMacroVariadicArgument):
yield IGNORE(location=(loc(peek(token_seq)) or l)),
_ = error_if_not_value(token_seq, TOKENS.RIGHT_PARENTHESIS, location=l)
def get_args(tokens, symbol_table, func_type):
if peek_or_terminal(tokens) != TOKENS.RIGHT_PARENTHESIS: # check the arguments.
for arg_decl, arg in takewhile(
lambda args: not (isinstance(c_type(args[0]), VAListType) and args[1] is None),
izip_longest(arguments(func_type), argument_expression_list(tokens, symbol_table))
):
if arg is None:
raise ValueError('{l} Function call with not enough arguments specified.'.format(l=l))
elif arg_decl is None:
raise ValueError('{l} Function call with to many arguments specified'.format(l=loc(arg)))
elif not safe_type_coercion(c_type(arg), c_type(arg_decl)):
raise ValueError('{l} Function call, could not coerce argument from {f_type} to {t_type}'.format(
l=loc(arg), f_type=c_type(arg), t_type=c_type(arg_decl),
))
yield CastExpression(arg, c_type(arg if isinstance(c_type(arg_decl), VAListType) else arg_decl), loc(arg))
def init_declarator(tokens, symbol_table, base_type=CType(''), storage_class=None):
# : declarator ('=' assignment_expression or initializer)?
decl = set_core_type(symbol_table['__ declarator __'](tokens, symbol_table), base_type)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
decl = Definition(name(decl), c_type(decl), EmptyExpression(c_type(decl)), loc(decl), storage_class)
symbol_table[name(decl)] = decl # we have to add it to the symbol table for things like `int a = a;`
expr = initializer_or_assignment_expression(tokens, symbol_table)
# if declaration is an array type and the expression is of string_type then convert to initializer for parsing
if isinstance(c_type(decl), ArrayType) and isinstance(c_type(expr), StringType):
expr = Initializer(
enumerate(exp(expr)), ArrayType(c_type(c_type(expr)), len(c_type(expr)), loc(expr)), loc(expr)
)
decl.initialization = parse_initializer(expr, decl) if isinstance(expr, Initializer) else expr
else:
symbol_table[name(decl)] = decl = Declaration(name(decl), c_type(decl), loc(decl))
return decl
def _func_macro_arguments(line):
symbol_table = SymbolTable()
while peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS:
if peek(line) == TOKENS.ELLIPSIS:
arg = FunctionMacroVariadicArgument(IDENTIFIER('__VA_ARGS__', loc(consume(line))))
else:
arg = FunctionMacroArgument(error_if_not_type(consume(line, EOFLocation), (IDENTIFIER, KEYWORD)))
if peek_or_terminal(line) == TOKENS.ELLIPSIS:
arg = FunctionMacroVariadicArgument(IDENTIFIER(arg, loc(consume(line))))
symbol_table[arg] = arg # check for duplicate argument name
yield arg # if ok add to the rest ...
if isinstance(arg, FunctionMacroVariadicArgument): # if variadic argument break ...
break
# consume expected comma if we don't see a right parenthesis ...
_ = peek(line, TOKENS.RIGHT_PARENTHESIS) != TOKENS.RIGHT_PARENTHESIS \
and error_if_not_value(line, TOKENS.COMMA, loc(arg))
def parse_struct_members(tokens, symbol_table):
declarator = symbol_table['__ declarator __']
location, members = loc(consume(tokens)), OrderedDict()
while peek(tokens, TOKENS.RIGHT_BRACE) != TOKENS.RIGHT_BRACE:
type_spec = specifier_qualifier_list(tokens, symbol_table)
while peek(tokens, TOKENS.SEMICOLON) != TOKENS.SEMICOLON:
decl = declarator(tokens, symbol_table)
set_core_type(decl, type_spec)
if name(decl) in members:
raise ValueError('{l} Duplicate struct member {name} previous at {at}'.format(
l=loc(decl), name=name(decl), at=loc(members[name(decl)])
))
members[name(decl)] = decl
_ = peek_or_terminal(tokens) != TOKENS.SEMICOLON and error_if_not_value(tokens, TOKENS.COMMA)
_ = error_if_not_value(tokens, TOKENS.SEMICOLON)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACE)
return members
def type_specifier(tokens, symbol_table, default=no_default):
"""
: 'void'
| ['signed' | 'unsigned'] 'char' | ['signed' | 'unsigned'] 'short'
| ['signed' | 'unsigned'] 'int' | ['signed' | 'unsigned'] 'long'
| 'float' | 'double'
| struct_specifier
| union_specifier
| enum_specifier
| TYPE_NAME
"""
token = peek_or_terminal(tokens)
if token in rules(type_specifier):
return rules(type_specifier)[token](tokens, symbol_table)
elif isinstance(symbol_table.get(token, token), CType):
return symbol_table[token](loc(consume(tokens)))
elif default is not no_default:
return default
raise ValueError('{l} Expected type_specifier or TYPE_NAME got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
))
