def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(
takewhile(
lambda i: isinstance(i, Allocate) and isinstance(
opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def composite_specifier(
tokens,
symbol_table,
obj_type=StructType,
member_parse_func=parse_struct_members,
terminal=object()
):
"""
: 'composite type' IDENTIFIER
| 'composite type' IDENTIFIER '{' members '}'
| 'composite type' '{' members '}'
"""
location = loc(consume(tokens))
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE: # anonymous composite ...
return obj_type(None, member_parse_func(tokens, symbol_table), location)
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
obj = symbol_table.get(obj_type.get_name(peek(tokens)), obj_type(consume(tokens), None, location))
# some composites are bit tricky such as Struct/Union ...
# since any of its members may contain itself as a reference, so we'll add the type to
# the symbol table before adding the members ...
# TODO: make types immutable, right now they are being shared.
if symbol_table.get(obj.name, terminal) is terminal:
symbol_table[name(obj)] = obj
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
obj.members = member_parse_func(tokens, symbol_table)
return obj
raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')
))
def body(stmnt, symbol_table, end_switch):
symbol_table = push(symbol_table)
stack, statement = imap(symbol_table.__getitem__, ('__ stack __', '__ statement __'))
symbol_table['__ break __'] = (end_switch, stack.stack_pointer)
symbol_table['__ switch __'] = True
allocation_table = [] # create an allocation table to update stack before jump in case of nested definitions
switch_body_instrs = []
cases = {'default': Offset(end_switch, loc(stmnt))}
for instr in statement(stmnt.statement, symbol_table):
if isinstance(getattr(instr, 'case', None), CaseStatement):
start = Pass(loc(instr))
allocation_table.append(
chain(
(start,),
update_stack(stmnt.stack.stack_pointer, instr.case.stack.stack_pointer, loc(instr)),
relative_jump(Offset(instr, loc(instr)), loc(instr)),
)
)
cases[error_if_not_type(exp(exp(instr.case)), (int, long, str))] = Offset(start, loc(instr))
del instr.case
switch_body_instrs.append(instr)
max_switch_value = 2**(8*size_arrays_as_pointers(c_type(exp(stmnt)))) - 1
for instr in jump_table(loc(stmnt), cases, allocation_table, max_switch_value, switch_body_instrs):
yield instr
_ = pop(symbol_table)
def composite_specifier(tokens,
symbol_table,
obj_type=StructType,
member_parse_func=parse_struct_members,
terminal=object()):
"""
: 'composite type' IDENTIFIER
| 'composite type' IDENTIFIER '{' members '}'
| 'composite type' '{' members '}'
"""
location = loc(consume(tokens))
if peek_or_terminal(
tokens) == TOKENS.LEFT_BRACE: # anonymous composite ...
return obj_type(None, member_parse_func(tokens, symbol_table),
location)
if isinstance(peek_or_terminal(tokens), IDENTIFIER):
obj = symbol_table.get(obj_type.get_name(peek(tokens)),
obj_type(consume(tokens), None, location))
# some composites are bit tricky such as Struct/Union ...
# since any of its members may contain itself as a reference, so we'll add the type to
# the symbol table before adding the members ...
# TODO: make types immutable, right now they are being shared.
if symbol_table.get(obj.name, terminal) is terminal:
symbol_table[name(obj)] = obj
if peek_or_terminal(tokens) == TOKENS.LEFT_BRACE:
obj.members = member_parse_func(tokens, symbol_table)
return obj
raise ValueError('{l} Expected IDENTIFIER or LEFT_BRACE got {got}'.format(
l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')))
def get_args(tokens, symbol_table, func_type):
if peek_or_terminal(
tokens) != TOKENS.RIGHT_PARENTHESIS: # check the arguments.
for arg_decl, arg in takewhile(
lambda args: not (isinstance(c_type(args[0]), VAListType) and
args[1] is None),
izip_longest(arguments(func_type),
argument_expression_list(tokens, symbol_table))):
if arg is None:
raise ValueError(
'{l} Function call with not enough arguments specified.'.
format(l=l))
elif arg_decl is None:
raise ValueError(
'{l} Function call with to many arguments specified'.
format(l=loc(arg)))
elif not safe_type_coercion(c_type(arg), c_type(arg_decl)):
raise ValueError(
'{l} Function call, could not coerce argument from {f_type} to {t_type}'
.format(
l=loc(arg),
f_type=c_type(arg),
t_type=c_type(arg_decl),
))
yield CastExpression(
arg,
c_type(arg if isinstance(c_type(arg_decl), VAListType
) else arg_decl), loc(arg))
def _merge_tokens(tokens):
for token in imap(consume, repeat(tokens)):
if token == TOKENS.PP:
token = IGNORE()
while peek_or_terminal(tokens) == TOKENS.PP and consume(
tokens):
new_token_source = token + consume(tokens, IGNORE())
new_tokens = tokenize(
imap(
Str, new_token_source,
imap(
Location,
repeat(
loc(token).file_name,
len(new_token_source)),
repeat(loc(token).line_number),
count(loc(token).column_number),
)))
token = next(new_tokens, IGNORE())
terminal_token = next(new_tokens, terminal)
if terminal_token is not terminal:
raise ValueError(
'{l} token pasting generated more than one token {t} {e}'
.format(l=loc(token), t=token, e=terminal_token))
if token == TOKENS.PP:
token = IGNORE()
yield token
def set_binaries(symbol):
assert not symbol.binaries and isinstance(symbol, object_file.Data)
instrs = starmap(Byte, repeat((0, loc(symbol)), symbol.size))
first_instr = next(instrs, terminal)
symbol.first_element = Pass(loc(symbol)) if first_instr is terminal else first_instr # zero sized decl use Pass
symbol.binaries = chain((symbol.first_element,), instrs)
return symbol.binaries
def init_declarator(tokens,
symbol_table,
base_type=CType(''),
storage_class=None):
# : declarator ('=' assignment_expression or initializer)?
decl = set_core_type(
symbol_table['__ declarator __'](tokens, symbol_table), base_type)
if peek_or_terminal(tokens) == TOKENS.EQUAL and consume(tokens):
decl = Definition(name(decl), c_type(decl),
EmptyExpression(c_type(decl)), loc(decl),
storage_class)
symbol_table[name(
decl
)] = decl # we have to add it to the symbol table for things like `int a = a;`
expr = initializer_or_assignment_expression(tokens, symbol_table)
# if declaration is an array type and the expression is of string_type then convert to initializer for parsing
if isinstance(c_type(decl), ArrayType) and isinstance(
c_type(expr), StringType):
expr = Initializer(
enumerate(exp(expr)),
ArrayType(c_type(c_type(expr)), len(c_type(expr)), loc(expr)),
loc(expr))
decl.initialization = parse_initializer(expr, decl) if isinstance(
expr, Initializer) else expr
else:
symbol_table[name(decl)] = decl = Declaration(name(decl), c_type(decl),
loc(decl))
return decl
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(takewhile(lambda i: isinstance(i, Allocate) and isinstance(opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def compound_assignment(expr, symbol_table):
assert all(imap(isinstance, imap(c_type, (left_exp(expr), right_exp(expr))), repeat(NumericType)))
assert not isinstance(c_type(left_exp(expr)), ArrayType)
if isinstance(left_exp(expr), IdentifierExpression) and \
base_c_type(c_type(left_exp(expr))) == base_c_type(c_type(right_exp(expr))) and \
size(c_type(left_exp(expr))) == size(c_type(right_exp(expr))):
# check that both operands are of the same kind (integral vs numeric) and have the same size ...
return simple_numeric_assignment_no_casting(expr, symbol_table, rules(compound_assignment)[oper(expr)])
max_type = max(imap(c_type, (left_exp(expr), right_exp(expr)))) # cast to largest type.
expression = symbol_table['__ expression __']
left_instrs = cast( # cast to max_type
patch_comp_left_instrs(expression(left_exp(expr), symbol_table), loc(expr), size(c_type(left_exp(expr)))),
c_type(left_exp(expr)),
max_type,
loc(expr),
)
right_instrs = cast(expression(right_exp(expr), symbol_table), c_type(right_exp(expr)), max_type, loc(expr))
return patch_comp_assignment(
cast( # Cast the result back, swap the value and the destination address call set to save.
rules(compound_assignment)[oper(expr)](
left_instrs,
right_instrs,
loc(expr),
(max_type, c_type(left_exp(expr)), c_type(right_exp(expr)))
),
max_type,
c_type(expr),
loc(expr)
),
c_type(expr),
loc(expr),
)
def const_string_expr(expr): # strings are embedded ...
data = static_binaries(expr)
_initial_data = peek(data) # there should be at least one char, '\0'
_push = push(Address(_initial_data, loc(expr)), loc(expr))
return chain(
relative_jump(Offset(peek(_push, loc(expr)), loc(expr)), loc(expr)),
consume_all(data, _push))
def copy_token(token, new_location=LocationNotSet):
if hasattr(token, 'suffix'):
return token.__new__(token.__class__,
str(token),
new_location or loc(token),
suffix=suffix(token))
return token.__new__(token.__class__, str(token), new_location
or loc(token))
def get_declaration_or_definition(decl, storage_class):
_ = initialization(decl) and isinstance(storage_class, Extern) and raise_error(
'{l} {ident} has both initialization expr and extern storage class'.format(l=loc(decl), ident=name(decl)))
if isinstance(c_type(decl), (FunctionType, StructType)) and not name(decl) or isinstance(storage_class, Extern):
return Declaration(name(decl), c_type(decl), loc(decl), storage_class)
return Definition(name(decl), c_type(decl), initialization(decl), loc(decl), storage_class or Auto(loc(decl)))
def unary_operator(tokens, symbol_table):
operator = consume(tokens)
if operator == TOKENS.LOGICAL_AND:
return AddressOfLabelExpression(
error_if_not_type(consume(tokens, ''), IDENTIFIER),
void_pointer_type(loc(operator)), loc(operator))
cast_exp = symbol_table['__ cast_expression __'](tokens, symbol_table)
return rules(unary_operator)[operator](cast_exp, operator)
def get_type(suffix_str, location):
_type, _eval = IntegerType(location,
unsigned=unsigned_suffix in suffix_str), int
_type, _eval = (long_suffix in suffix_str and LongType(
_type, loc(_type), unsigned(_type))) or _type, long
_type, _eval = (long_long_suffix in suffix_str and LongType(
_type, loc(_type), unsigned(_type))) or _type, long
return _type, _eval
def subtraction_expression(_exp):
if is_binary_pointer_expression(_exp): # if subtracting two pointers, then the return type is a LongType
return BinaryExpression(
left_exp(_exp), oper(_exp), right_exp(_exp),
LongType(LongType(location=loc(_exp)), unsigned=True, location=loc(_exp)),
location=loc(_exp)
)
return _exp
def set_binaries(symbol):
assert not symbol.binaries and isinstance(symbol, object_file.Data)
instrs = starmap(Byte, repeat((0, loc(symbol)), symbol.size))
first_instr = next(instrs, terminal)
symbol.first_element = Pass(
loc(symbol)
) if first_instr is terminal else first_instr # zero sized decl use Pass
symbol.binaries = chain((symbol.first_element, ), instrs)
return symbol.binaries
def case_statement(stmnt, symbol_table):
try:
_ = symbol_table['__ switch __']
except KeyError as _:
raise ValueError('{l} case statement outside switch statement'.format(l=loc(stmnt)))
initial_instr = Pass(loc(stmnt))
stmnt.stack = deepcopy(symbol_table['__ stack __']) # case statements may be placed in nested compound statements.
initial_instr.case = stmnt
return chain((initial_instr,), symbol_table['__ statement __'](stmnt.statement, symbol_table))
def convert_declaration_to_definition(decl):
_ = isinstance(decl, FunctionDefinition) and raise_error(
'{l} Nested function definitions are not allowed.'.format(l=loc(decl)))
# Non Function declaration without storage class is set to auto
if type(decl) is Declaration and not isinstance(
c_type(decl), FunctionType) and decl.storage_class is not Extern:
decl = Definition( # all non-function-declarations within compound statements are definitions ...
name(decl), c_type(decl), EmptyExpression(c_type(decl), loc(decl)),
loc(decl), decl.storage_class or Auto(loc(decl)))
return decl
def inc_dec(value, expr, symbol_table):
return symbol_table['__ expression __'](
CompoundAssignmentExpression(
exp(expr), TOKENS.PLUS_EQUAL,
ConstantExpression(
value, IntegerType(loc(expr), unsigned=unsigned(c_type(expr))),
loc(expr)),
c_type(expr)(loc(expr)), loc(expr)),
symbol_table,
)
def identifier_expression(expr, symbol_table):
# Defaults to Load, assignment expression will update it to set.
dec = symbol_table[name(expr)]
if isinstance(
c_type(dec),
(FunctionType,
ArrayType)): # Function/Array Types are nothing more than addresses.
return dec.load_address(loc(expr))
return load(dec.load_address(loc(expr)),
size_arrays_as_pointers(c_type(expr)), loc(expr))
def inc_dec(expr, symbol_table):
assert not isinstance(c_type(expr), ArrayType) and isinstance(
c_type(expr), IntegralType)
value = rules(inc_dec)[type(expr)]
if isinstance(c_type(expr), PointerType):
value *= size_extended(c_type(c_type(expr)))
return get_postfix_update(size(c_type(expr)))(all_but_last(
symbol_table['__ expression __'](exp(expr), symbol_table), Loads,
loc(expr)), value, loc(expr))
def numeric_operator(value, expr, symbol_table):
return symbol_table['__ expression __'](
BinaryExpression(
ConstantExpression(value, IntegerType(loc(expr)), loc(expr)),
TOKENS.STAR,
exp(expr),
max(c_type(exp(expr)), IntegerType(loc(expr)))(loc(expr)),
loc(expr),
),
symbol_table,
)
def inc_dec(value, expr, symbol_table):
return symbol_table['__ expression __'](
CompoundAssignmentExpression(
exp(expr),
TOKENS.PLUS_EQUAL,
ConstantExpression(value, IntegerType(loc(expr), unsigned=unsigned(c_type(expr))), loc(expr)),
c_type(expr)(loc(expr)),
loc(expr)
),
symbol_table,
)
def continue_statement(stmnt, symbol_table):
try:
instr, stack_pointer = symbol_table['__ continue __']
except KeyError as _:
raise ValueError('{l} continue statement outside loop statement, could not calc jump addr'.format(
l=loc(stmnt)
))
return chain(
update_stack(symbol_table['__ stack __'].stack_pointer, stack_pointer, loc(stmnt)),
relative_jump(Offset(instr, loc(stmnt)), loc(stmnt))
)
def numeric_operator(value, expr, symbol_table):
return symbol_table['__ expression __'](
BinaryExpression(
ConstantExpression(value, IntegerType(loc(expr)), loc(expr)),
TOKENS.STAR,
exp(expr),
max(c_type(exp(expr)), IntegerType(loc(expr)))(loc(expr)),
loc(expr),
),
symbol_table,
)
def while_statement(stmnt, symbol_table):
start_of_loop, end_of_loop = imap(Pass, repeat(loc(stmnt), 2))
return chain(
(start_of_loop,),
get_jump_false(size_arrays_as_pointers(c_type(exp(stmnt))))(
symbol_table['__ expression __'](exp(stmnt), symbol_table), Offset(end_of_loop, loc(end_of_loop)), loc(end_of_loop)
),
loop_body(stmnt.statement, symbol_table, start_of_loop, end_of_loop),
relative_jump(Offset(start_of_loop, loc(start_of_loop)), loc(end_of_loop)),
(end_of_loop,)
)
def binary_exp(expr):
if not all(imap(isinstance, (left_exp(expr), right_exp(expr)), repeat(ConstantExpression))):
return expr
exp_type = max(imap(c_type, (left_exp(expr), right_exp(expr))))(loc(expr))
l_exp, r_exp, location = exp(left_exp(expr)), exp(right_exp(expr)), loc(expr)
# noinspection PyUnresolvedReferences '1 + 2 - 3 * 7 / 4'
return binary_exp.rules[oper(expr)](
expr=expr, left_exp=l_exp, right_exp=r_exp, location=location, exp_type=exp_type
)
def assign(expr, symbol_table):
expression = symbol_table['__ expression __']
return set_instr(
chain(
cast(expression(right_exp(expr), symbol_table), c_type(right_exp(expr)), c_type(expr), loc(expr)),
# remove default Load instruction, emit Set instruction ...
all_but_last(expression(left_exp(expr), symbol_table), Loads, loc(expr)),
),
size_arrays_as_pointers(c_type(expr)), # get the size exp returns an array make sure its treated as pointer ...
loc(expr),
)
def set_addresses(instrs, addresses=None): # assign addresses ...
def default_address_gen():
previous_address = 0
_ = (yield)
while True: # TODO: calculate address based on size of previous instr (for now all 1 word)
_ = (yield previous_address)
previous_address += word_size
addresses = addresses or default_address_gen()
_ = next(addresses)
references = []
for instr in instrs:
instr.address = addresses.send(
instr) # calculate next address based on current value ...
yield instr
if type(instr) is Address and type(instr.obj) not in {int, long}:
references.append((instr, instr, None))
for operand_index, o in enumerate(operns(instr, ())):
if isinstance(o, (Address, Offset)) and type(
o.obj) not in {int, long}:
references.append((instr, o, operand_index))
# replace immutable int type by mutable type.
if type(o) in {int, long}:
o = Word(o, loc(instr))
instr[operand_index] = o
elif type(o) is float:
o = Double(o, loc(instr))
instr[operand_index] = o # update instruction operand.
o.address = addresses.send(o)
yield o
# At this point all address have being generated and resolve() should have checked or set first_element on Refs ...
# so update addresses replace Address(obj=instr) by Address(obj=address or offset)
# print '\n'.join(imap(str, references))
for instr, operand, operand_index in references:
if not hasattr(operand.obj, 'address'):
print instr, loc(operand), ' no_address ---> ', id(
operand.obj), operand.obj
if isinstance(operand, Offset):
operand.obj = operand.obj.address - (
instr.address +
(2 * word_size if isinstance(instr, RelativeJump) else 0))
else:
operand.obj = operand.obj.address
# Update instruction operand ...
if isinstance(instr, Instruction):
instr[operand_index] = operand.__class__(operand.obj, loc(operand))
def do_while_statement(stmnt, symbol_table):
start_of_loop, end_of_loop = Pass(loc(stmnt)), Pass(loc(stmnt))
yield start_of_loop # do while loops contain the update expression after their body ...
for instr in loop_body(stmnt.statement, symbol_table, start_of_loop, end_of_loop):
yield instr
for instr in get_jump_true(size_arrays_as_pointers(c_type(exp(stmnt))))(
symbol_table['__ expression __'](exp(stmnt), symbol_table),
Offset(start_of_loop, loc(start_of_loop)),
loc(start_of_loop)
):
yield instr
yield end_of_loop
def subtraction_expression(_exp):
if is_binary_pointer_expression(
_exp
): # if subtracting two pointers, then the return type is a LongType
return BinaryExpression(left_exp(_exp),
oper(_exp),
right_exp(_exp),
LongType(LongType(location=loc(_exp)),
unsigned=True,
location=loc(_exp)),
location=loc(_exp))
return _exp
def static_definition(stmnt, symbol_table):
def load_address(self, location):
return push(Address(self._initial_data, location), location)
data = static_def_binaries(stmnt, (Pass(loc(stmnt)),))
stmnt._initial_data = peek(data)
stmnt.end_of_data = Pass(loc(stmnt))
stmnt.load_address = bind_load_address_func(load_address, stmnt)
symbol_table[declarations.name(stmnt)] = stmnt
return chain( # jump over embedded data ...
relative_jump(Offset(stmnt.end_of_data, loc(stmnt)), loc(stmnt)), consume_all(data), (stmnt.end_of_data,)
)
def defaults(ctype, designations=None):
designations = designations or OrderedDict()
if isinstance(ctype, NumericType):
designations[0] = ConstantExpression(0, ctype(loc(ctype)), loc(ctype))
elif isinstance(ctype, ArrayType):
for index in xrange(len(ctype)):
designations[index] = defaults(c_type(ctype))
elif isinstance(ctype, StructType):
for offset, member_name in enumerate(ctype.members):
designations[offset] = defaults(c_type(ctype.members[member_name]))
else:
raise ValueError('{l} Unable to generate default expression for ctype {c}'.format(l=loc(ctype), c=ctype))
return designations
def dimensions(tokens):
while peek(tokens) == TOKENS.LEFT_BRACKET:
location = loc(consume(tokens))
if peek(tokens) == TOKENS.RIGHT_BRACKET:
size = None
else:
const_exp = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(const_exp), IntegralType)
if exp(const_exp) < 0:
raise ValueError('{l} array size is negative'.format(l=loc(const_exp)))
size = exp(const_exp)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACKET)
yield size, location
def set_default_initializer(initializer, default_values):
# Complete ArrayTypes with initializer containing a single non designated expression assign the value through out
if isinstance(c_type(default_values), ArrayType) \
and not isinstance(initializer[0], DesignatedExpression) \
and len(initializer) == 1 \
and c_type(default_values).length is not None:
initializer = Initializer(
enumerate(repeat(initializer[0], len(c_type(default_values)))),
c_type(default_values)(loc(initializer)),
loc(initializer)
)
exhaust(imap(parse_designated_expr, initializer_desig_exprs(initializer, default_values), repeat(default_values)))
return default_values
def exclamation_operator(expr, symbol_table):
if not isinstance(c_type(exp(expr)), NumericType):
raise ValueError('{l} exclamation operator only supports numeric types got {g}'.format(
l=loc(expr), g=c_type(exp(expr))
))
assert not isinstance(c_type(exp(expr)), ArrayType)
expression = symbol_table['__ expression __']
return get_compare(size_arrays_as_pointers(c_type(exp(expr))))(
expression(exp(expr), symbol_table),
expression(ConstantExpression(0, c_type(exp(expr))(loc(expr)), loc(expr),), symbol_table),
loc(expr),
(load_zero_flag(loc(expr)),)
)
def convert_declaration_to_definition(decl):
_ = isinstance(decl, FunctionDefinition) and raise_error(
'{l} Nested function definitions are not allowed.'.format(l=loc(decl)))
# Non Function declaration without storage class is set to auto
if type(decl) is Declaration and not isinstance(c_type(decl), FunctionType) and decl.storage_class is not Extern:
decl = Definition( # all non-function-declarations within compound statements are definitions ...
name(decl),
c_type(decl),
EmptyExpression(c_type(decl), loc(decl)),
loc(decl),
decl.storage_class or Auto(loc(decl))
)
return decl
def __init__(self, c_decl, body):
_ = error_if_not_type(c_type(c_decl), FunctionType)
if not all(isinstance(arg, Declarator) for arg in c_type(c_decl)):
raise ValueError(
'{l} FunctionDef must have concrete declarators as params'.
format(l=loc(c_type(c_decl))))
if not isinstance(body, CompoundStatement):
raise ValueError(
'{l} FunctionDef body is not a compound statement, got {g}'.
format(l=loc(c_decl), g=body))
super(FunctionDefinition,
self).__init__(name(c_decl), c_type(c_decl), body, loc(c_decl),
c_decl.storage_class)
def get_declaration_or_definition(decl, storage_class):
_ = initialization(decl) and isinstance(
storage_class, Extern) and raise_error(
'{l} {ident} has both initialization expr and extern storage class'
.format(l=loc(decl), ident=name(decl)))
if isinstance(c_type(decl),
(FunctionType, StructType)) and not name(decl) or isinstance(
storage_class, Extern):
return Declaration(name(decl), c_type(decl), loc(decl), storage_class)
return Definition(name(decl), c_type(decl), initialization(decl),
loc(decl), storage_class or Auto(loc(decl)))
def set_addresses(instrs, addresses=None): # assign addresses ...
def default_address_gen():
previous_address = 0
_ = (yield)
while True: # TODO: calculate address based on size of previous instr (for now all 1 word)
_ = (yield previous_address)
previous_address += word_size
addresses = addresses or default_address_gen()
_ = next(addresses)
references = []
for instr in instrs:
instr.address = addresses.send(instr) # calculate next address based on current value ...
yield instr
if type(instr) is Address and type(instr.obj) not in {int, long}:
references.append((instr, instr, None))
for operand_index, o in enumerate(operns(instr, ())):
if isinstance(o, (Address, Offset)) and type(o.obj) not in {int, long}:
references.append((instr, o, operand_index))
# replace immutable int type by mutable type.
if type(o) in {int, long}:
o = Word(o, loc(instr))
instr[operand_index] = o
elif type(o) is float:
o = Double(o, loc(instr))
instr[operand_index] = o # update instruction operand.
o.address = addresses.send(o)
yield o
# At this point all address have being generated and resolve() should have checked or set first_element on Refs ...
# so update addresses replace Address(obj=instr) by Address(obj=address or offset)
# print '\n'.join(imap(str, references))
for instr, operand, operand_index in references:
if not hasattr(operand.obj, 'address'):
print instr, loc(operand), ' no_address ---> ', id(operand.obj), operand.obj
if isinstance(operand, Offset):
operand.obj = operand.obj.address - (
instr.address + (2 * word_size if isinstance(instr, RelativeJump) else 0)
)
else:
operand.obj = operand.obj.address
# Update instruction operand ...
if isinstance(instr, Instruction):
instr[operand_index] = operand.__class__(operand.obj, loc(operand))
def dimensions(tokens):
while peek(tokens) == TOKENS.LEFT_BRACKET:
location = loc(consume(tokens))
if peek(tokens) == TOKENS.RIGHT_BRACKET:
size = None
else:
const_exp = constant_expression(tokens, symbol_table)
_ = error_if_not_type(c_type(const_exp), IntegralType)
if exp(const_exp) < 0:
raise ValueError(
'{l} array size is negative'.format(l=loc(const_exp)))
size = exp(const_exp)
_ = error_if_not_value(tokens, TOKENS.RIGHT_BRACKET)
yield size, location
def get_binary_expression(tokens, symbol_table, l_exp, right_exp_func, exp_type):
operator = consume(tokens)
r_exp = right_exp_func(tokens, symbol_table)
exp_type = max(imap(error_if_not_type, imap(c_type, (l_exp, r_exp)), repeat(exp_type)))
if operator in LOGICAL_OPERATIONS:
exp_type = logical_type
if operator not in supported_operations(c_type(l_exp)):
raise ValueError('{l} ctype {g} does not support {o}'.format(l=loc(l_exp), g=c_type(l_exp), o=operator))
if operator not in supported_operations(c_type(r_exp)):
raise ValueError('{l} ctype {g} does not support {o}'.format(l=loc(r_exp), g=c_type(r_exp), o=operator))
return BinaryExpression(l_exp, operator, r_exp, exp_type(location=loc(operator)), loc(operator))
def update_default_value(desig_expr, default_values):
ctype, desig, expr = c_type(default_values), designation(desig_expr), exp(desig_expr)
if desig >= len(default_values):
logger.warning(
'{0} Excess element {1} {2} in initializer, it will be ignored ... '.format(
loc(desig_expr), desig, expr
))
else:
_ = (not safe_type_coercion(c_type(expr), c_type(default_values[desig]))) and raise_error(
'{l} Unable to coerce from {f} to {t}'.format(l=loc(expr), f=c_type(expr), t=c_type(default_values[desig]))
)
update_func = update_composite_type_initializer \
if isinstance(ctype, (StructType, ArrayType)) else update_scalar_type_initializer
update_func(desig_expr, default_values)
def case_statement(stmnt, symbol_table):
try:
_ = symbol_table['__ switch __']
except KeyError as _:
raise ValueError(
'{l} case statement outside switch statement'.format(l=loc(stmnt)))
initial_instr = Pass(loc(stmnt))
stmnt.stack = deepcopy(
symbol_table['__ stack __']
) # case statements may be placed in nested compound statements.
initial_instr.case = stmnt
return chain((initial_instr, ),
symbol_table['__ statement __'](stmnt.statement,
symbol_table))
def relational_operators(expr, symbol_table):
max_type = max(imap(c_type, (left_exp(expr), right_exp(expr))))
expression = symbol_table['__ expression __']
if isinstance(max_type, ArrayType):
max_type = PointerType(c_type(max_type), loc(max_type))
left_instrs = expression(left_exp(expr), symbol_table)
right_instrs = expression(right_exp(expr), symbol_table)
return rules(relational_operators)[oper(expr)](
cast(left_instrs, c_type(left_exp(expr)), max_type, loc(expr)),
cast(right_instrs, c_type(right_exp(expr)), max_type, loc(expr)),
loc(expr),
(c_type(expr), c_type(left_exp(expr)), c_type(right_exp(expr))),
)
def set_default_initializer(initializer, default_values):
# Complete ArrayTypes with initializer containing a single non designated expression assign the value through out
if isinstance(c_type(default_values), ArrayType) \
and not isinstance(initializer[0], DesignatedExpression) \
and len(initializer) == 1 \
and c_type(default_values).length is not None:
initializer = Initializer(
enumerate(repeat(initializer[0], len(c_type(default_values)))),
c_type(default_values)(loc(initializer)), loc(initializer))
exhaust(
imap(parse_designated_expr,
initializer_desig_exprs(initializer, default_values),
repeat(default_values)))
return default_values
def element_selection(expr, symbol_table):
instrs = symbol_table['__ expression __'](left_exp(expr), symbol_table)
# if we are loading the structure then just remove the Load instr, calculate the elements offset and Load that elem
if is_load(instrs):
return element_instrs(
c_type(left_exp(expr)), right_exp(expr), loc(expr), load_instrs=all_but_last(instrs, Loads, loc(expr))
)
struct_size, member_size = size(c_type(left_exp(expr))), size_arrays_as_pointers(c_type(expr))
addr_instrs = add( # calculate the loaded structured members address
load_stack_pointer(loc(expr)),
push(struct_member_offset(c_type(left_exp(expr)), right_exp(expr)), loc(expr)),
loc(expr)
)
return chain( # copy the element then move it to the base of the structure and deallocate everything else
set_instr(
chain( # load the value in question if its not an array (which is just an address ...)
(addr_instrs if isinstance(c_type(expr), ArrayType) else load(addr_instrs, member_size, loc(expr))),
add(load_stack_pointer(loc(expr)), push((struct_size + member_size), loc(expr)), loc(expr)),
),
member_size,
loc(expr)
),
allocate(-(struct_size - member_size), loc(expr)) # deallocate structure and copied member (set leaves value)
)
def cast_exp(expr):
if c_type(expr) == c_type(exp(expr)):
return exp(expr)
if isinstance(exp(expr), ConstantExpression) and not isinstance(c_type(exp(expr)), ArrayType):
to_type = c_type(expr)
expr = exp(expr)
location = loc(expr)
if isinstance(expr, EmptyExpression):
return EmptyExpression(to_type, loc(expr))
if isinstance(to_type, IntegralType):
return ConstantExpression(int(exp(expr)), to_type(location), location)
if isinstance(to_type, FloatType):
return ConstantExpression(float(exp(expr)), to_type(location), location)
return expr
def declarations(tokens, symbol_table):
# storage_class_specifier? type_name? init_declarator_list (';' or compound_statement) # declaration
storage_class_specifier, specifier_qualifier_list, statement = imap(
symbol_table.__getitem__,
('__ storage_class_specifier __', '__ specifier_qualifier_list __',
'__ statement __'))
storage_class = storage_class_specifier(tokens, symbol_table)
base_type = specifier_qualifier_list(tokens, symbol_table)
expecting_token = TOKENS.SEMICOLON
if peek_or_terminal(tokens) == TOKENS.SEMICOLON:
yield EmptyDeclaration(loc(consume(tokens)), storage_class)
elif peek_or_terminal(tokens) is terminal:
raise_error(
'{l} Expected TOKENS.COMMA TOKENS.EQUAL TOKENS.SEMICOLON TOKENS.LEFT_BRACE got `{got}`'
.format(l=loc(peek(tokens, EOFLocation)), got=peek(tokens, '')))
else:
for dec in init_declarator_list(tokens,
symbol_table,
base_type=base_type,
storage_class=storage_class):
dec.storage_class = storage_class
if isinstance(
storage_class, TypeDef
): # init_declarator_list adds the symbol as a decl to symbol_table
symbol_table[name(dec)] = (symbol_table.pop(
name(dec)) or 1) and c_type(dec) # replace dec by ctype
elif peek_or_terminal(
tokens) == TOKENS.LEFT_BRACE and not error_if_not_type(
c_type(dec), FunctionType):
symbol_table = push(symbol_table)
symbol_table.update(
chain(
imap(
lambda a: (
name(a), a
), # add non variable list parameters to the symbol table ...
ifilterfalse(
lambda c: isinstance(c_type(c), VAListType),
c_type(dec))),
(('__ RETURN_TYPE __', c_type(c_type(dec))),
('__ LABELS __', SymbolTable()))))
yield FunctionDefinition(dec,
next(statement(tokens, symbol_table)))
expecting_token = (pop(symbol_table) or 1) and ''
else:
yield dec
expecting_token = TOKENS.SEMICOLON
_ = expecting_token and error_if_not_value(tokens, expecting_token)
def _return(tokens, symbol_table):
location = loc(consume(tokens))
ret_type = symbol_table['__ RETURN_TYPE __']
ret_exp = EmptyExpression(VoidType(location), location)
if peek_or_terminal(tokens) != TOKENS.SEMICOLON:
ret_exp = symbol_table['__ expression __'](tokens, symbol_table)
if not isinstance(ret_exp, EmptyExpression) and isinstance(ret_type, VoidType):
raise ValueError('{l} void-function returning a value ...'.format(l=loc(ret_exp)))
if not safe_type_coercion(c_type(ret_exp), ret_type):
raise ValueError('{l} Unable to coerce from {f} to {t}'.format(l=loc(ret_exp), f=c_type(ret_exp), t=ret_type))
return ReturnStatement(ret_exp, location)
def defaults(ctype, designations=None):
designations = designations or OrderedDict()
if isinstance(ctype, NumericType):
designations[0] = ConstantExpression(0, ctype(loc(ctype)), loc(ctype))
elif isinstance(ctype, ArrayType):
for index in xrange(len(ctype)):
designations[index] = defaults(c_type(ctype))
elif isinstance(ctype, StructType):
for offset, member_name in enumerate(ctype.members):
designations[offset] = defaults(c_type(ctype.members[member_name]))
else:
raise ValueError(
'{l} Unable to generate default expression for ctype {c}'.format(
l=loc(ctype), c=ctype))
return designations
def arithmetic_and_bitwise_operators(expr, symbol_table):
expression = symbol_table['__ expression __']
left_instrs = expression(left_exp(expr), symbol_table)
right_instrs = expression(right_exp(expr), symbol_table)
to_type = c_type(expr)
if isinstance(to_type, ArrayType):
to_type = PointerType(c_type(to_type), loc(c_type(expr)))
return rules(arithmetic_and_bitwise_operators)[oper(expr)](
cast(left_instrs, c_type(left_exp(expr)), to_type, loc(expr)),
cast(right_instrs, c_type(right_exp(expr)), to_type, loc(expr)),
loc(expr),
(to_type, c_type(left_exp(expr)), c_type(right_exp(expr))),
)
def ternary_expression(expr, symbol_table):
if_false_instr, end_of_conditional_instr = Pass(loc(expr)), Pass(loc(expr))
expression = symbol_table['__ expression __']
return chain(
get_jump_false(size_arrays_as_pointers(c_type(exp(expr))))(
expression(exp(expr), symbol_table),
Offset(if_false_instr, loc(expr)),
loc(expr)
),
expression(left_exp(expr), symbol_table),
relative_jump(Offset(end_of_conditional_instr, loc(end_of_conditional_instr)), loc(expr)),
(if_false_instr,),
expression(right_exp(expr), symbol_table),
(end_of_conditional_instr,),
)