def pre_processor(char_stream, location): # returns pre_processing symbol or #identifier ...
values = consume(char_stream)
if peek_or_terminal(char_stream) == TOKENS.NUMBER_SIGN: # token concatenation symbol ...
values += consume(char_stream)
else:
_ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
values += ''.join(takewhile(letters.__contains__, char_stream))
return rules(pre_processor).get(values, IDENTIFIER)(values, location)
def pre_processor(
char_stream,
location): # returns pre_processing symbol or #identifier ...
values = consume(char_stream)
if peek_or_terminal(
char_stream
) == TOKENS.NUMBER_SIGN: # token concatenation symbol ...
values += consume(char_stream)
else:
_ = exhaust(takewhile({' ', '\t', '\a'}.__contains__, char_stream))
values += ''.join(takewhile(letters.__contains__, char_stream))
return rules(pre_processor).get(values, IDENTIFIER)(values, location)
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(
takewhile(
lambda i: isinstance(i, Allocate) and isinstance(
opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def remove_allocation(instrs):
"""
optimize 1 or more sequence of allocations ...
take their sum and if zero replace with the next instruction in case this one is referenced.
other wise do one allocation and remove rest
replace allocate 1 with POP, which only requires a single address translation vs 2 (instr, oprn) for allocate.
"""
alloc_instrs = tuple(takewhile(lambda i: isinstance(i, Allocate) and isinstance(opern(i), (int, long)), instrs))
if not alloc_instrs: # Operand must be non-primitive type (Address) ... must wait for its value.
yield consume(instrs)
else:
total = sum(imap(opern, alloc_instrs))
if total: # non-zero allocates changes the state of the stack.
if total in pop_instrs:
new_instr = next(pop_instrs[total](loc(alloc_instrs[0])))
elif len(alloc_instrs) != 1:
new_instr = alloc_instrs[0]
else:
new_instr = Allocate(loc(alloc_instrs[-1]), total)
yield replace_instrs(new_instr, alloc_instrs)
else: # stack remains unchanged, get next instruction for referencing, it one exists ...
if peek_or_terminal(instrs) is terminal:
yield replace_instr(Pass(loc(alloc_instrs[-1])), alloc_instrs)
else:
replace_instrs(peek(instrs), alloc_instrs)
def get_line(
values
): # get all the tokens on the current line, being that preprocessor works on a line-by-line basis
return takewhile(
lambda token, initial_line_number=line_number(peek(
values)): initial_line_number == line_number(token),
values) if peek_or_terminal(values) is not terminal else iter(())
def remove_pass(instrs):
""" replace 1 or more sequences of Pass by the next non-Pass instruction or Pass instruction """
pass_instrs = tuple(takewhile(lambda instr: isinstance(instr, Pass), instrs))
if peek_or_terminal(instrs) is terminal:
yield replace_instrs(pass_instrs[-1], pass_instrs[:-1])
else:
replace_instrs(peek(instrs), pass_instrs)
def optimize(instrs, level=zero_level_optimization):
global new_instructions, old_instructions, deleted_instructions
new_instructions = {}
old_instructions = defaultdict(list)
deleted_instructions = []
return update_instruction_references(
chain.from_iterable(imap(level, takewhile(peek, repeat(instrs)))))
def string_literal(tokens):
location = loc(peek(tokens)) # join adjacent strings into a single string ...
token = ''.join(takewhile(lambda t: type(t) is STRING, tokens)) + '\0'
return ConstantExpression(
imap(char_literal, imap(iter, token)),
StringType(len(token), location),
location
)
def type_qualifiers(tokens, _, defaults=None): # : ('const' or volatile or *args)*
values = set(takewhile(rules(type_qualifiers).__contains__, tokens))
const, volatile = imap(values.__contains__, (TOKENS.CONST, TOKENS.VOLATILE))
if not values and not defaults:
raise ValueError('{l} Expected TOKENS.CONST or TOKEN.VOLATILE got {g}'.format(
l=loc(peek(tokens, EOFLocation)), g=peek(tokens, '')
))
return const or defaults[0], volatile or defaults[1]
def number(char_stream, hexadecimal_chars={'x', 'X'}):
initial_char, _digits = '', digits
if peek_or_terminal(char_stream) == digit(0):
initial_char = consume(char_stream)
if peek_or_terminal(char_stream) in hexadecimal_chars:
initial_char += consume(char_stream)
_digits = hexadecimal_digits
return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))
def remove_pass(instrs):
""" replace 1 or more sequences of Pass by the next non-Pass instruction or Pass instruction """
pass_instrs = tuple(
takewhile(lambda instr: isinstance(instr, Pass), instrs))
if peek_or_terminal(instrs) is terminal:
yield replace_instrs(pass_instrs[-1], pass_instrs[:-1])
else:
replace_instrs(peek(instrs), pass_instrs)
def number(char_stream, hexadecimal_chars={'x', 'X'}):
initial_char, _digits = '', digits
if peek_or_terminal(char_stream) == digit(0):
initial_char = consume(char_stream)
if peek_or_terminal(char_stream) in hexadecimal_chars:
initial_char += consume(char_stream)
_digits = hexadecimal_digits
return initial_char + ''.join(takewhile(_digits.__contains__, char_stream))
def exhaust_remaining_blocks(token_seq):
exhaust(
imap(
apply,
imap(
rules(exhaust_remaining_blocks).__getitem__,
takewhile(TOKENS.PENDIF.__ne__, imap(peek,
repeat(token_seq)))),
repeat((token_seq, ))))
def get_repositioned_line(char_seq, location): # get next line ...
while not isinstance(peek(char_seq), NewLineStr):
char = consume(char_seq)
if char == '\\' and isinstance(peek(char_seq), NewLineStr):
_ = exhaust(takewhile(lambda token: isinstance(token, NewLineStr), char_seq))
for char in get_repositioned_line(char_seq, location):
yield char
else:
yield Str(char, location)
def merge_lines(char_seq):
while True:
char = consume(char_seq)
if char == '\\' and isinstance(peek(char_seq), NewLineStr): # if current char is \ followed by end of line seq
_ = exhaust(takewhile(lambda token: isinstance(token, NewLineStr), char_seq))
for char in get_repositioned_line(char_seq, loc(char)):
yield char
else:
yield char
def get_block(token_seq, terminating_with={TOKENS.PENDIF}):
return chain.from_iterable(
imap(
apply,
imap(
rules(get_block).__getitem__,
takewhile(lambda token: token not in terminating_with,
imap(peek, repeat(token_seq)))), repeat(
(token_seq, ))))
def type_qualifiers(tokens,
_,
defaults=None): # : ('const' or volatile or *args)*
values = set(takewhile(rules(type_qualifiers).__contains__, tokens))
const, volatile = imap(values.__contains__,
(TOKENS.CONST, TOKENS.VOLATILE))
if not values and not defaults:
raise ValueError(
'{l} Expected TOKENS.CONST or TOKEN.VOLATILE got {g}'.format(
l=loc(peek(tokens, EOFLocation)), g=peek(tokens, '')))
return const or defaults[0], volatile or defaults[1]
def exhaust_remaining_blocks(token_seq):
exhaust(
imap(
apply,
imap(
rules(exhaust_remaining_blocks).__getitem__,
takewhile(TOKENS.PENDIF.__ne__, imap(peek, repeat(token_seq)))
),
repeat((token_seq,))
)
)
def get_block(token_seq, terminating_with={TOKENS.PENDIF}):
return chain.from_iterable(
imap(
apply,
imap(
rules(get_block).__getitem__,
takewhile(lambda token: token not in terminating_with, imap(peek, repeat(token_seq)))
),
repeat((token_seq,))
)
)
def get_repositioned_line(char_seq, location): # get next line ...
while not isinstance(peek(char_seq), NewLineStr):
char = consume(char_seq)
if char == '\\' and isinstance(peek(char_seq), NewLineStr):
_ = exhaust(
takewhile(lambda token: isinstance(token, NewLineStr),
char_seq))
for char in get_repositioned_line(char_seq, location):
yield char
else:
yield Str(char, location)
def merge_lines(char_seq):
while True:
char = consume(char_seq)
if char == '\\' and isinstance(
peek(char_seq), NewLineStr
): # if current char is \ followed by end of line seq
_ = exhaust(
takewhile(lambda token: isinstance(token, NewLineStr),
char_seq))
for char in get_repositioned_line(char_seq, loc(char)):
yield char
else:
yield char
def get_newline_if_possible(char_seq):
return ''.join(takewhile(
lambda c, new_line=iter(linesep), _seq=char_seq: c == next(new_line) and (consume(_seq) or 1),
imap(peek, repeat(char_seq))
))
def get_line(tokens):
return iter(()) if peek_or_terminal(tokens) is terminal else takewhile(
lambda t, current_line_number=line_number(peek(tokens)): line_number(
t) == current_line_number,
tokens)
def keyword_or_identifier(char_stream, location):
values = ''.join(takewhile(alpha_numeric.__contains__, char_stream))
return rules(keyword_or_identifier).get(values, IDENTIFIER)(values, location)
def suffix(char_stream):
return ''.join(takewhile(letters.__contains__, char_stream))
def keyword_or_identifier(char_stream, location):
values = ''.join(takewhile(alpha_numeric.__contains__, char_stream))
return rules(keyword_or_identifier).get(values, IDENTIFIER)(values,
location)
def get_line(values): # get all the tokens on the current line, being that preprocessor works on a line-by-line basis
return takewhile(
lambda token, initial_line_number=line_number(peek(values)): initial_line_number == line_number(token),
values
) if peek_or_terminal(values) is not terminal else iter(())
def format_token_seq(token_seq):
return chain.from_iterable(imap(format_line, imap(get_line, takewhile(peek, repeat(token_seq)))))
def standard_lib_file_path(token_seq, _):
file_path = consume(token_seq) and ''.join(
takewhile(TOKENS.GREATER_THAN.__ne__, token_seq))
_ = error_if_not_value(token_seq, TOKENS.GREATER_THAN)
return file_path
def suffix(char_stream):
return ''.join(takewhile(letters.__contains__, char_stream))
def get_line(tokens):
return iter(()) if peek_or_terminal(tokens) is terminal else takewhile(
lambda t, current_line_number=line_number(peek(tokens)): line_number(t) == current_line_number,
tokens
)
def standard_lib_file_path(token_seq, _):
file_path = consume(token_seq) and ''.join(takewhile(TOKENS.GREATER_THAN.__ne__, token_seq))
_ = error_if_not_value(token_seq, TOKENS.GREATER_THAN)
return file_path
def string_literal(tokens):
location = loc(
peek(tokens)) # join adjacent strings into a single string ...
token = ''.join(takewhile(lambda t: type(t) is STRING, tokens)) + '\0'
return ConstantExpression(imap(char_literal, imap(iter, token)),
StringType(len(token), location), location)
def format_token_seq(token_seq):
return chain.from_iterable(
imap(format_line, imap(get_line, takewhile(peek, repeat(token_seq)))))
def optimize(instrs, level=zero_level_optimization):
global new_instructions, old_instructions, deleted_instructions
new_instructions = {}
old_instructions = defaultdict(list)
deleted_instructions = []
return update_instruction_references(chain.from_iterable(imap(level, takewhile(peek, repeat(instrs)))))
def get_newline_if_possible(char_seq):
return ''.join(
takewhile(lambda c, new_line=iter(linesep), _seq=char_seq: c == next(
new_line) and (consume(_seq) or 1),
imap(peek, repeat(char_seq))))