def test_variadic_macros(self):
code = """
#define foo(...) __VA_ARGS__
foo()
foo(1,2,3,4)
"""
self.assertEqual('1 , 2 , 3 , 4', ' '.join(preprocess(tokenize(source(code)))))
def evaluate(self, code):
self.cpu, self.mem = CPU(), VirtualMemory()
symbol_table = SymbolTable(
(
('__ LABELS __', SymbolTable()),
('__ GOTOS __', defaultdict(list)),
('__ stack __', Stack()),
('__ expression __', expression),
('__ statement __', statement)
)
)
load(
set_addresses(
chain(
emitter.statement(
next(parser.statement(
preprocess(tokenize(source(code))), SymbolTable((('__ LABELS __', SymbolTable()),)))
),
symbol_table
),
(Halt('__EOP__'),)
)
),
self.mem,
)
evaluate(self.cpu, self.mem)
def _merge_tokens(tokens):
for token in imap(consume, repeat(tokens)):
if token == TOKENS.PP:
token = IGNORE()
while peek_or_terminal(tokens) == TOKENS.PP and consume(
tokens):
new_token_source = token + consume(tokens, IGNORE())
new_tokens = tokenize(
imap(
Str, new_token_source,
imap(
Location,
repeat(
loc(token).file_name,
len(new_token_source)),
repeat(loc(token).line_number),
count(loc(token).column_number),
)))
token = next(new_tokens, IGNORE())
terminal_token = next(new_tokens, terminal)
if terminal_token is not terminal:
raise ValueError(
'{l} token pasting generated more than one token {t} {e}'
.format(l=loc(token), t=token, e=terminal_token))
if token == TOKENS.PP:
token = IGNORE()
yield token
def test_tokenize(self):
code = """
#include "stdio.h"
int main()
{
float a = .0;// this is a comment.
char b = 'a';/* this is
a multi-line comment */
int a = 1 / 2;
return 0;
}
"""
new_tokens = tokenize(source(code))
tokens = (PRE_PROCESSING_SYMBOL('#include'), STRING('stdio.h'),
KEYWORD('int'), IDENTIFIER('main'), SYMBOL('('), SYMBOL(')'),
SYMBOL('{'), KEYWORD('float'), IDENTIFIER('a'), SYMBOL('='),
FLOAT('.0'), SYMBOL(';'), KEYWORD('char'), IDENTIFIER('b'),
SYMBOL('='), CHAR('a'), SYMBOL(';'), KEYWORD('int'),
IDENTIFIER('a'), SYMBOL('='), INTEGER(1), SYMBOL('/'),
INTEGER(2), SYMBOL(';'), KEYWORD('return'), INTEGER('0'),
SYMBOL(';'), SYMBOL('}'))
for exp_token, got_token in izip(tokens, new_tokens):
self.assertEqual(exp_token, got_token)
self.assertEqual(type(exp_token), type(got_token))
def test_simple_declarations(self):
code = """
int foo(int (*)(void *, int[1]));
int a, b = 1, c = 5 + 5;
"""
decs = translation_unit(preprocess(tokenize(source(code))))
int_type = IntegerType('')
exp_decls = [
Declaration(
'foo',
FunctionType(int_type, [AbstractDeclarator(PointerType(FunctionType(
int_type,
[
AbstractDeclarator(PointerType(VoidType(''), ''), ''),
AbstractDeclarator(ArrayType(int_type, 1, ''), ''),
],
'',
), ''), '')], ''),
'',
),
Declaration('a', int_type, ''),
Definition('b', int_type, ConstantExpression(1, int_type, ''), '', None),
Definition('c', int_type, ConstantExpression(10, int_type, ''), '', None)
]
for got_dec, exp_dec in izip(decs, exp_decls):
self.assertEqual(got_dec, exp_dec)
def test_variadic_named_macros(self):
code = """
#define foo(args...) args
foo()
foo(1,2,3,4)
"""
self.assertEqual('1 , 2 , 3 , 4', ' '.join(preprocess(tokenize(source(code)))))
def test_executable(self):
source_codes = """
extern int b;
int main()
{
b = 10;
return 0;
}
""", 'int b;'
mem = VirtualMemory()
cpu = CPU()
symbol_table = SymbolTable()
load(
set_addresses(
resolve(
executable(
chain.from_iterable(emit(parse(preprocess(tokenize(source(code))))) for code in source_codes),
symbol_table
),
symbol_table
),
),
mem
)
evaluate(cpu, mem)
def test_bug(self):
code = """
#define vector_type void *
#define __copy_element__(src, dest, element_type, index) (*(element_type *)(dest + index) = *(element_type *)(src + index))
__copy_element__(src, dest, vector_type, index);
"""
self.assertEqual('( * ( void * * ) ( dest + index ) = * ( void * * ) ( src + index ) ) ;',
' '.join(preprocess(tokenize(source(code)))))
def evaluate_expr(self, code):
self.mem, self.cpu = VirtualMemory(), CPU()
load(
set_addresses(
chain(emit_expression(parser.expression(preprocess(tokenize(source(code))))), (Halt('__EOP__'),)),
),
self.mem
)
evaluate(self.cpu, self.mem)
def evaluate_expr(self, code):
self.mem, self.cpu = VirtualMemory(), CPU()
load(
set_addresses(
chain(
emit_expression(
parser.expression(preprocess(tokenize(source(code))))),
(Halt('__EOP__'), )), ), self.mem)
evaluate(self.cpu, self.mem)
def test_nested_func_macros(self):
code = """
#define size(block) (((block_type *)block)->size)
block_type *blocks = freed_blocks[size(block)];
"""
self.assertEqual(
'block_type * blocks = freed_blocks [ ( ( ( block_type * ) block ) -> size ) ] ;',
' '.join(preprocess(tokenize(source(code))))
)
def test_nested_obj_macros(self):
code = """
#define is_aligned(address, size) (!(((unsigned long)address) & (size - 1)))
#define vector_type void *
is_aligned((dest + numb - sizeof(vector_type)), sizeof(vector_type))
"""
self.assertEqual(
'( ! ( ( ( unsigned long ) ( dest + numb - sizeof ( void * ) ) ) & ( sizeof ( void * ) - 1 ) ) )',
' '.join(preprocess(tokenize(source(code)))),
)
def evaluate(self, code):
symbol_table, self.cpu, self.mem = SymbolTable(), CPU(), VirtualMemory()
load(
set_addresses(
resolve(executable(emit(parse(preprocess(tokenize(source(code))))), symbol_table), symbol_table)
),
self.mem,
)
evaluate(self.cpu, self.mem)
def INCLUDE(token_seq, macros):
line = get_line(token_seq)
file_path = consume(line) and get_rule(INCLUDE, peek_or_terminal(line), hash_funcs=(type, identity))(line, macros)
search_paths = (os.getcwd(),)
_ = error_if_not_empty(line)
return chain(
macros['__ preprocess __'](
tokenize(load(file_path, chain(macros['__ include_dirs __'], search_paths))),
macros
),
)
def INCLUDE(token_seq, macros):
line = get_line(token_seq)
file_path = consume(line) and get_rule(
INCLUDE, peek_or_terminal(line), hash_funcs=(type, identity))(line,
macros)
search_paths = (os.getcwd(), )
_ = error_if_not_empty(line)
return chain(
macros['__ preprocess __'](tokenize(
load(file_path, chain(macros['__ include_dirs __'],
search_paths))), macros), )
def test_pre_processor(self):
code = """
#define a 1
#define b(a) a
#if defined(b) - 1 + defined a
b(a)
#else
0
#endif
"""
self.assert_token_seq(('1',), preprocess(tokenize(source(code))))
def symbols(file_name, include_dirs=(), optimizer=identity):
if isinstance(file_name, str) and file_name.endswith('.o.p'):
with open(file_name) as file_obj:
symbol_table = pickle.load(file_obj)
symbol_seq = symbol_table.itervalues()
else:
symbol_seq = optimizer(
emitter.emit(
parser.parse(
preprocessor.preprocess(tokenizer.tokenize(
loader.load(file_name)),
include_dirs=include_dirs))))
return symbol_seq
def test_multiple_nested_func_macros(self):
code = """
#define size(block) (((block_type *)block)->size)
#define set_size(block, value) (size(block) = value)
#define next(block) (((block_type *)block)->next)
#define set_next(block, value) (next(block) = value)
set_next(next(blocks), block);
"""
self.assertEqual(
'( ( ( ( block_type * ) ( ( ( block_type * ) blocks ) -> next ) ) -> next ) = block ) ;',
' '.join(preprocess(tokenize(source(code))))
)
def test_compound(self):
code = """
int main()
{
int a = 1 + 2;
label:
for (a = 0; a < 3; a++)
a;
goto label;
return 0;
}
"""
got_values = translation_unit(preprocess(tokenize(source(code))))
self.assert_(isinstance(next(got_values), FunctionDefinition))
def symbols(file_name, include_dirs=(), optimizer=identity):
if isinstance(file_name, str) and file_name.endswith('.o.p'):
with open(file_name) as file_obj:
symbol_table = pickle.load(file_obj)
symbol_seq = symbol_table.itervalues()
else:
symbol_seq = optimizer(
emitter.emit(
parser.parse(
preprocessor.preprocess(tokenizer.tokenize(loader.load(file_name)), include_dirs=include_dirs)
)
)
)
return symbol_seq
def test_binary_expressions(self):
expressions = (
('1 + 2 - 3 * 7 / 4', -2),
('1 >> 2 + 3 << 2 + 5', 0),
('1 || 0 + 4 - 10.5 / 1 && 0', 1),
('(1 + 2) * (1.0 + 2.0 + 3)', 18.0),
('(1 + 2) * 3 - 3 / 1 >> 2 * 1 << 2', 4),
)
for raw_exp, expected_result in expressions:
tokens = tokenize(source(raw_exp))
actual_result = expression(preprocess(tokens))
self.assertEqual(
expected_result, actual_result.exp,
'Raw exp {exp}, expected {e}, got {g}'.format(
exp=raw_exp, e=expected_result, g=actual_result.exp))
def test_binary_expressions(self):
expressions = (
('1 + 2 - 3 * 7 / 4', -2),
('1 >> 2 + 3 << 2 + 5', 0),
('1 || 0 + 4 - 10.5 / 1 && 0', 1),
('(1 + 2) * (1.0 + 2.0 + 3)', 18.0),
('(1 + 2) * 3 - 3 / 1 >> 2 * 1 << 2', 4),
)
for raw_exp, expected_result in expressions:
tokens = tokenize(source(raw_exp))
actual_result = expression(preprocess(tokens))
self.assertEqual(
expected_result, actual_result.exp,
'Raw exp {exp}, expected {e}, got {g}'.format(exp=raw_exp, e=expected_result, g=actual_result.exp)
)
def test_nested_conditional_blocks(self):
code = """
#if 0
#ifdef foo
int a = 0;
#else
#endif
#elif 1
#ifndef alice
#if 0
#elif 1
ok
#else
int a = 0;
#endif
#endif
#endif
"""
self.assert_token_seq(('ok',), preprocess(tokenize(source(code))))
def evaluate(self, code):
self.cpu, self.mem = CPU(), VirtualMemory()
symbol_table = SymbolTable(
(('__ LABELS __', SymbolTable()),
('__ GOTOS __', defaultdict(list)), ('__ stack __', Stack()),
('__ expression __', expression), ('__ statement __', statement)))
load(
set_addresses(
chain(
emitter.statement(
next(
parser.statement(
preprocess(tokenize(source(code))),
SymbolTable(
(('__ LABELS __', SymbolTable()), )))),
symbol_table), (Halt('__EOP__'), ))),
self.mem,
)
evaluate(self.cpu, self.mem)
def test_composite_initializer(self):
code = """
struct {int a[100]; struct {int b; void *ptr; char sub[10];} values[10];} g = {
.values[0] = {-1, (void *)-1},
.a[0 ... 99] = 1,
.values[1 ... 5] = {.b = 5},
.values[6 ... 9].ptr = 1,
.values[1].sub = {1},
.values[1].sub[0] = 'a'
};
"""
expr = next(parse(preprocess(tokenize(source(code))))).initialization
a = expr[0]
values = expr[1]
self.assertEqual(len(a), 100)
self.assertEqual(len(values), 10)
a_expr = ConstantExpression(1, integer_type)
for e in imap(next, imap(lambda v: v.itervalues(), a.itervalues())):
self.assertEqual(e, a_expr)
self.assertEqual(values[0][0][0], ConstantExpression(-1, integer_type))
self.assertEqual(values[0][1][0],
ConstantExpression(-1, void_pointer_type))
b_expr = ConstantExpression(5, integer_type)
for index in xrange(1, 6):
self.assertEqual(values[index][0][0], b_expr)
self.assertEqual(values[index][1][0],
EmptyExpression(integer_type))
ptr_expr = ConstantExpression(1, void_pointer_type)
for index in xrange(6, 10):
self.assertEqual(values[index][1][0], ptr_expr)
self.assertEqual(values[index][0][0],
EmptyExpression(void_pointer_type))
self.assertEqual(values[1][2][0][0],
ConstantExpression(ord('a'), char_type))
def test_executable(self):
source_codes = """
extern int b;
int main()
{
b = 10;
return 0;
}
""", 'int b;'
mem = VirtualMemory()
cpu = CPU()
symbol_table = SymbolTable()
load(
set_addresses(
resolve(
executable(
chain.from_iterable(
emit(parse(preprocess(tokenize(source(code)))))
for code in source_codes), symbol_table),
symbol_table), ), mem)
evaluate(cpu, mem)
def test_composite_initializer(self):
code = """
struct {int a[100]; struct {int b; void *ptr; char sub[10];} values[10];} g = {
.values[0] = {-1, (void *)-1},
.a[0 ... 99] = 1,
.values[1 ... 5] = {.b = 5},
.values[6 ... 9].ptr = 1,
.values[1].sub = {1},
.values[1].sub[0] = 'a'
};
"""
expr = next(parse(preprocess(tokenize(source(code))))).initialization
a = expr[0]
values = expr[1]
self.assertEqual(len(a), 100)
self.assertEqual(len(values), 10)
a_expr = ConstantExpression(1, integer_type)
for e in imap(next, imap(lambda v: v.itervalues(), a.itervalues())):
self.assertEqual(e, a_expr)
self.assertEqual(values[0][0][0], ConstantExpression(-1, integer_type))
self.assertEqual(values[0][1][0], ConstantExpression(-1, void_pointer_type))
b_expr = ConstantExpression(5, integer_type)
for index in xrange(1, 6):
self.assertEqual(values[index][0][0], b_expr)
self.assertEqual(values[index][1][0], EmptyExpression(integer_type))
ptr_expr = ConstantExpression(1, void_pointer_type)
for index in xrange(6, 10):
self.assertEqual(values[index][1][0], ptr_expr)
self.assertEqual(values[index][0][0], EmptyExpression(void_pointer_type))
self.assertEqual(values[1][2][0][0], ConstantExpression(ord('a'), char_type))
def test_compound_literal(self):
code = "(struct {int a; char b[10]; struct {double b;} foo;}){.a = 1, .b = {1}, .foo.b = 1.3};"
expr = expression(preprocess(tokenize(source(code))))
self.assertIsInstance(expr, CompoundLiteral)
def test_default_basic_type_initializer(self):
code = "int b = {};"
expr = next(parse(preprocess(tokenize(source(code))))).initialization
self.assertEqual(EmptyExpression(integer_type), expr[0])
def test_failed_expectation(self):
code = "int a!"
self.assertRaises(
ValueError, list,
self.single_declaration(preprocess(tokenize(source(code)))))
def preprocess_file(input_file, include_dirs):
return preprocessor.preprocess(tokenizer.tokenize(loader.load(input_file)),
include_dirs=include_dirs)
def test_missing_semicolon(self):
code = "int a"
self.assertRaises(
ValueError, list,
self.single_declaration(preprocess(tokenize(source(code)))))
def test_extra_values(self):
code = """
#define foo
#undef foo error!
"""
self.assertRaises(ValueError, list, preprocess(tokenize(source(code))))
def preprocess_file(input_file, include_dirs):
return preprocessor.preprocess(tokenizer.tokenize(loader.load(input_file)), include_dirs=include_dirs)
def test_compound_literal(self):
code = "(struct {int a; char b[10]; struct {double b;} foo;}){.a = 1, .b = {1}, .foo.b = 1.3};"
expr = expression(preprocess(tokenize(source(code))))
self.assertIsInstance(expr, CompoundLiteral)
def test_missing_semicolon(self):
code = "int a"
self.assertRaises(ValueError, list, self.single_declaration(preprocess(tokenize(source(code)))))
def test_default_basic_type_initializer(self):
code = "int b = {};"
expr = next(parse(preprocess(tokenize(source(code))))).initialization
self.assertEqual(EmptyExpression(integer_type), expr[0])
def test_token_concatenation(self):
code = """
#define foo(a, b) a ## b ## 5 ## a
foo(h, ello)
"""
self.assert_token_seq(('hello5h',), preprocess(tokenize(source(code))))
def test_stringification(self):
code = """
#define foo(e) #e
foo(hello)
"""
self.assert_token_seq(('hello',), preprocess(tokenize(source(code))))
def test_extra_values(self):
code = """
#define foo
#undef foo error!
"""
self.assertRaises(ValueError, list, preprocess(tokenize(source(code))))
def test_failed_expectation(self):
code = "int a!"
self.assertRaises(ValueError, list, self.single_declaration(preprocess(tokenize(source(code)))))
def test_bug_1(self):
code = """
#define LEFT_NODE(tree) tree->left
#define VALUE LEFT_NODE
VALUE(VALUE(old_leaf)) = old_leaf;
"""
self.assertEqual('old_leaf -> left -> left = old_leaf ;', ' '.join(preprocess(tokenize(source(code)))))
