def from_code(self, code: str, parser=None):
"""
Takes (partial) C code as a string and returns the AST node that belongs to it. Wraps everything in a block.
Note: This function may not understand all code strings and shall be used with care. But, it understands
everything that is either directly valid C code, or partial code that can be put into a function body or as a
condition block (e.g. if and while conditions).
:param code: The C code.
:param parser: If you have already created a parser, you can hint it here. No need to create multiple parsers.
:return: The corresponding AST compound.
:rtype: c_ast.Compound or None
"""
# TODO this is a bit hacky, trying to wrap the code s.t. it is hopefully valid to allow for partial parsing.
if not parser: parser = GnuCParser()
try:
ast = parser.parse(code)
return c_ast.Compound(ast.ext)
except:
try:
wrapped_code = "void f() { " + code + " }"
ast = parser.parse(wrapped_code)
return ast.ext[0].body
except:
try:
wrapped_code = "void f() { if(" + code + ") {} }"
ast = parser.parse(wrapped_code)
return c_ast.Compound(
[ast.ext[0].body.block_items[0].cond])
except Exception as e:
print(e)
return None
def parse_c(filename, header_dir, use_fakes=False):
global __opts
if use_fakes:
fake_include_dir = fake_headers()
if fake_include_dir:
__opts.append('-I%s' % fake_include_dir)
parse_error_mo = re.compile(r'([/\w\.\-]+):(\d+):(\d+): before: .*')
ast = None
retry_parse = True
while retry_parse:
source = __preprocess(filename, header_dir)
parser = CParser()
try:
ast = parser.parse(source, filename)
retry_parse = False
except ParseError as e:
m = parse_error_mo.match(e.args[0])
if not m:
raise
opt = __guess_symbol(m.group(1), int(m.group(2)), int(m.group(3)))
if opt and opt not in __opts:
__opts.append(opt)
else:
raise
return ast
def generate_org_fw_functions(src_files, org_functions_file,
org_functions_header, org_functions_lcs, symtab,
base_elf, gen_dir):
"""
determines the helper functions required to generate and writes them
:param src_files: list of source code files
:param org_functions_file: destination file for generated c-code
:param org_functions_header: destination file for generated header code
:param org_functions_lcs: destination file for generated linker symbol definitions
:param symtab: symbol table
:param base_elf: base firmware ELF file
:param gen_dir: destination directory for generated (intermediate) files
"""
# determine required original function helpers
req_org_funcs = Set()
for src_file in src_files:
text = preprocess_file(src_file, 'hexagon-cpp',
r'-DFW_WRAPPER="/dev/null"')
try: # pycparserext >= 2016.2
parser = GnuCParser(taboutputdir=gen_dir)
except TypeError:
parser = GnuCParser()
ast = parser.parse(text, src_file)
v = FuncCallVisitor()
v.visit(ast)
req_org_funcs = req_org_funcs.union(v.get_req_org_funcs())
# generate and write the functions
write_functions(req_org_funcs, org_functions_file, org_functions_header,
org_functions_lcs, symtab, base_elf)
def generate_patch_list(symtab, fw_wrapper, src_files, gen_dir):
"""
generates a list of patches needed to be applied in order to patch the ELF file
:param symtab: symbol table
:param fw_wrapper: firmware wrapper header file
:param src_files: list of patch source code files
:param gen_dir: destination directory for generated (intermediate) files
"""
patches = []
for filename in src_files:
text = preprocess_file(filename, 'hexagon-cpp',
'-DFW_WRAPPER="' + fw_wrapper + '"')
try: # pycparserext >= 2016.2
parser = GnuCParser(taboutputdir=gen_dir)
except TypeError:
parser = GnuCParser()
ast = parser.parse(text, filename)
v = FuncDefVisitor()
v.visit(ast)
patches.extend(v.patches)
return patches
def preprocess(path):
text = preprocess_file(path)
cparser = GnuCParser()
ast = cparser.parse(text, path)
generator = ext_c_generator.GnuCGenerator()
with open(path, "w") as o:
o.write(generator.visit(ast))
return ast
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_gnu_statement_expression():
src = """
int func(int a) {
return (int)({; ; *(int*)&a;});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_funky_header_code_5():
src=""" void do_foo(void) __asm(__STRING(do_foo));"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_funky_header_code_5():
src = """ void do_foo(void) __asm(__STRING(do_foo));"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_gnu_statement_expression():
src = """
int func(int a) {
return (int)({; ; *(int*)&a;});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_nesty_c_declarator():
src = """
struct a {
int *b[1][1];
};
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_func_ret_ptr_decl_attribute():
src = """
extern void* memcpy(const void* src, const void *dst, int len) __attribute__((unused));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_func_ret_ptr_decl_attribute():
src = """
extern void* memcpy(const void* src, const void *dst, int len) __attribute__((unused));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_nesty_c_declarator():
src = """
struct a {
int *b[1][1];
};
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_empty_struct_declaration():
src = """
typedef struct Foo {
} Foo_t;
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_attributes():
src = """
int x[10] __attribute__((unused));
int y[20] __attribute((aligned(10)));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_asm_volatile_1():
src = """
void read_tsc(void) {
long val;
asm("rdtsc" : "=A" (val));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_asm_volatile_3():
src = """
void read_tsc(void) {
long fpenv;
asm("mtfsf 255,%0" :: "f" (fpenv));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print GnuCGenerator().visit(ast)
def test_empty_struct_declaration():
src = """
typedef struct Foo {
} Foo_t;
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_asm_volatile_2():
src = """
void read_tsc(void) {
long val;
asm volatile("rdtsc" : "=A" (val));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print GnuCGenerator().visit(ast)
def test_asm_volatile_3():
src = """
void read_tsc(void) {
long fpenv;
asm("mtfsf 255,%0" :: "f" (fpenv));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_attributes():
src = """
int x[10] __attribute__((unused));
int y[20] __attribute((aligned(10)));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def write_files(dir_list, header_include, out_dir, fn_names):
header_file = header_include.split('/')[-1]
is_valid_file = [
os.path.isfile(dir_name + header_file) for dir_name in dir_list
]
if not any(is_valid_file):
print "Skipped all functions in %s because it could not be found in %s." % (
header_file, dir_list)
return
else:
header_file = dir_list[is_valid_file.index(True)] + header_file
# pycparser utility function for preprocessing the file and
# getting all of the file as a string
text = preprocess_file(header_file)
# use pycparserext to parse the GNU C header file
p = GnuCParser()
ast = p.parse(text)
# loop over all functions requested
for fn in fn_names:
found = False
# loop over the ast
for node in ast.ext:
if isinstance(node, c_ast.FuncDef) or not isinstance(
node.type, c_ast.FuncDecl):
continue
# skip over functions with __noreturn__
if hasattr(node.type.type, 'attributes') and any([
e.name == "__noreturn__"
for e in node.type.type.attributes.exprs
]):
continue
if node.name != fn[0]:
continue
found = True
write_file(out_dir, header_include, node, fn[1], fn[2])
if not found:
print "The header file %s does not contain a method signature for %s" % (
header_file, fn[0])
def test_func_decl_attribute():
src = """
extern void int happy(void) __attribute__((unused));
int main()
{
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_lvalue_gnu_statement_expression():
src = """
int func(int a) {
int ret=(int)({; ; *(int*)&a;});
return ret;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def _round_trip_matches(src):
from pycparserext.ext_c_parser import GnuCParser
from pycparserext.ext_c_generator import GnuCGenerator
p = GnuCParser()
first_ast = p.parse(src)
gen = GnuCGenerator().visit(first_ast)
second_ast = p.parse(gen)
if not _compare_asts(first_ast, second_ast):
print('First AST:')
first_ast.show()
print('Generated code:')
print(gen)
print('Second AST:')
second_ast.show()
return False
return True
def test_func_decl_attribute():
src = """
extern void int happy(void) __attribute__((unused));
int main()
{
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_lvalue_gnu_statement_expression():
src = """
int func(int a) {
int ret=(int)({; ; *(int*)&a;});
return ret;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_const_ptr_func_arg():
src = """
const int *bar;
void foo(const int *bar, int * const baz);
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
src_str = GnuCGenerator().visit(ast)
assert src_str.count("*") == 3
print(src_str)
def test_const_ptr_func_arg():
src = """
const int *bar;
void foo(const int *bar, int * const baz);
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
src_str = GnuCGenerator().visit(ast)
assert src_str.count("*") == 3
print(src_str)
def test_funky_header_code():
src = """
extern __inline int __attribute__ ((__nothrow__)) __signbitf (float __x)
{
int __m;
__asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
return __m & 0x8;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
from pycparserext.ext_c_generator import GnuCGenerator
print GnuCGenerator().visit(ast)
def test_funky_header_code():
src = """
extern __inline int __attribute__ ((__nothrow__)) __signbitf (float __x)
{
int __m;
__asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
return __m & 0x8;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_empty_gnu_statement_expression():
# Incase, ASSERTS turn out to be empty statements
src = """
int func(int a) {
({
; ;
});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_empty_gnu_statement_expression():
# Incase, ASSERTS turn out to be empty statements
src = """
int func(int a) {
({
; ;
});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def generate_wrapper_lcs(filename, lcs_file, symtab_json_file, gen_dir):
"""
main linker script generation function
:param filename: firmware wrapper header
:param lcs_file: destination file
:param symtab_json_file: destination file for symbol table in JSON format
:param gen_dir: destination directory for generated (intermediate) files
"""
text = preprocess_file(filename, 'hexagon-cpp')
try: # pycparserext >= 2016.2
parser = GnuCParser(taboutputdir=gen_dir)
except TypeError:
parser = GnuCParser()
ast = parser.parse(text, filename)
v = DeclVisitor(lcs_file)
v.visit(ast)
json.dump(v.get_symtab(), open(symtab_json_file, 'w'))
def extract_signature(
func_name: str, body: str
) -> Optional[Tuple[str, List[Tuple[str, str]]]]:
signature = None
class FuncDefVisitor(c_ast.NodeVisitor):
def visit_FuncDeclExt(self, node):
nonlocal signature # bad
# so much badness below
def get_typ_str(node):
if isinstance(node, c_ast.PtrDecl):
return f"{' '.join(node.type.type.names)}*"
else:
return " ".join(node.type.names)
def get_arg_name(node):
if isinstance(node, c_ast.PtrDecl):
return node.type.declname
else:
return node.declname
if get_arg_name(node.children()[1][1]) == func_name:
return_typ = get_typ_str(node.children()[1][1])
params = node.children()[0][1]
signature = (
return_typ,
[(get_arg_name(p.type), get_typ_str(p.type)) for p in params]
if params
else [],
)
parser = GnuCParser()
ast = parser.parse(body)
v = FuncDefVisitor()
v.visit(ast)
# ast.show()
return signature
def generate_cdef():
"""Generate the cdef output file"""
include_v4l2_path = '/usr/include/linux'
#include_v4l2_path='.'
#out_file = path.join(HERE, 'v4l2', 'videodev2.cdef.h')
#out_packed_file = path.join(HERE, 'v4l2', 'videodev2.cdef_packed.h')
#header = path.join(include_v4l2_path, 'videodev2.h')
#header_parsed = path.join(HERE, 'v4l2', 'videodev2.h')
#enum_file = path.join(HERE, 'v4l2', 'v4l2_enums.py')
out_file = os.path.join(HERE, BUILDDIR, 'videodev2.cdef.h')
out_packed_file = os.path.join(HERE, BUILDDIR, 'videodev2.cdef_packed.h')
header = os.path.join(include_v4l2_path, 'videodev2.h')
header_parsed = os.path.join(HERE, BUILDDIR, 'videodev2.h')
enum_file = os.path.join(HERE, 'v4l2enums.py')
out = open(header_parsed, 'w+')
cpp_process = subprocess.Popen(
[
'cpp',
'-P',
#'-nostdinc',
'-I',
'fake-include',
header
],
stdout=out)
cpp_process.wait()
out.close()
headerin = open(header_parsed, 'r')
headersrc = headerin.read()
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(headersrc, filename=header_parsed)
# ast.show()
headerin.close()
out = open(out_file, 'w+')
out_packed = open(out_packed_file, 'w+')
enums = open(enum_file, 'w+')
enums.write('import enum\nimport v4l2\n')
from pycparserext.ext_c_generator import GnuCGenerator
g = GnuCGenerator()
i = 0
for nname, astnode in ast.children():
outthis = out
if type(astnode) == pycparser.c_ast.Decl:
#print('node', i)
#print(g.visit(astnode)+'\n')
for spec in astnode.funcspec:
if type(spec) == pycparserext.ext_c_parser.AttributeSpecifier:
for att in spec.exprlist.exprs:
if att.name == 'packed':
outthis = out_packed
if type(astnode.type) == pycparser.c_ast.Enum:
enumnode = astnode.type
enums.write('class ' + enumnode.name + '(enum.IntEnum):\n')
for el in enumnode.values.enumerators:
enums.write(' ' + el.name + ' = v4l2.' + el.name + '\n')
enums.write('\n')
if type(astnode.type) != pycparserext.ext_c_parser.FuncDeclExt:
#print(i, type(astnode.type))
outthis.write(g.generic_visit(astnode) + ';\n')
else:
# for parsing ioctl(...) decalaration
outthis.write(g.visit(astnode) + ';\n')
else:
outthis.write(g.visit(astnode) + ';\n')
#print('node', i)
#print(g.generic_visit(astnode)+'\n')
i += 1
out.flush()
out.close()
out_packed.flush()
out_packed.close()
enums.flush()
enums.close()
print('generate_cdef: generated\n ', out_file, ',\n ',
out_packed_file, '\n and', enum_file, 'from', header)
return out_file, out_packed_file
from pycparserext.ext_c_parser import GnuCParser p = GnuCParser() src=""" typedef int wchar_t; extern wchar_t *wcscpy (wchar_t *restrict __dest, const wchar_t *__restrict __src) __attribute ((nothrow , leaf)); """ p.parse(src).show()
def prepare_induction_step(input_file: str,
original_input_file: str = None,
readd_property_from_original: bool = False):
"""
Prepares the input C file for the execution of the induction step. It parses the code, havocs the main loop
variables and adds the property assumption to the beginning of the loop body. When finished, the C code is written
to a temporary working file which is then returned.
:param input_file: The input C file location to prepare.
:param original_input_file: The input C file location of the original input code.
:param readd_property_from_original: Whether the original input file should be read and the property readded from
this file to the input file.
:return: The location of the prepared C file for the induction step.
:rtype: str
"""
parser = GnuCParser()
with open(input_file) as file:
ast = parser.parse(file.read())
analyzer = CAnalyzer(ast)
transformer = CTransformer(ast)
# De-anonymizes aggregates as a first step, as this is a requirement for later analysis.
# transformer.deanonymize_aggregates()
# Identifies main components of the code.
try:
main_function = analyzer.identify_function(MAIN_FUNCTION_NAME)
main_loop = analyzer.identify_main_loop()
declarations = analyzer.identify_declarations_of_modified_variables(
main_loop.stmt, main_function)
if readd_property_from_original and original_input_file and input_file != original_input_file:
with open(original_input_file) as original_file:
# In case we sliced the input, we want to re-add the original property first, as Frama-C scrambles the
# if statement in such a way that it becomes unrecognizable for our property identification process.
transformer.add_property(
CAnalyzer(parser.parse(
original_file.read())).identify_property(), main_loop)
property = analyzer.identify_property()
except (NoMainLoopException, MultipleMainLoopsException,
UnidentifiableVariableTypeException) as err:
print(err)
sys.exit(1)
# Creates new code and functions to be added.
try:
svcomp_havoc_functions, havoc_block = transformer.create_havoc_block(
declarations)
except NonSvCompTypeException as err:
print(err)
sys.exit(1)
for svcomp_havoc_function in svcomp_havoc_functions:
transformer.insert(transformer.create_svcomp_function_declaration(
svcomp_havoc_function),
before=ast.ext[0])
conjunct_property = transformer.join_expression_list("&&", property)
negated_property = transformer.add_to_expression(conjunct_property, "!")
assume_property = transformer.create_function_call(
VERIFIER_ASSUME_FUNCTION_NAME, negated_property)
# Adds some code to emulate incremental BMC: Only check property if we are in the k-th main loop iteration.
if not VERIFIER_IS_INCREMENTAL:
k_initialization = transformer.from_code("const unsigned int k = 1;",
parser).block_items[0]
i_initialization = transformer.from_code("unsigned int i = 0;",
parser).block_items[0]
i_increment = transformer.from_code("i++;", parser).block_items[0]
k_property = transformer.add_to_expression(
property.exprs[0], "&&",
c_ast.ExprList(transformer.from_code("(i == k)").block_items))
transformer.replace_property(k_property)
transformer.insert(k_initialization, before=main_loop)
transformer.insert(i_initialization, before=main_loop)
transformer.insert(i_increment, before=main_loop.stmt.block_items[0])
# Inserts new code pieces.
transformer.insert(havoc_block, before=main_loop)
transformer.insert(assume_property, before=main_loop.stmt.block_items[0])
# Writes the transformed code to a temporary file.
output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".c")
output_file.write(bytes(GnuCGenerator().visit(ast), "utf-8"))
return output_file.name
def visit_AttributeSpecifier(self, n):
return ' __attribute__((' + self.visit(n.exprlist) + '))'
####################################################################################################
class MyCGenerator(AsmAndAttributesMixin, CGenerator):
pass
####################################################################################################
file_path = 'fitz-from-cpp.h'
with open(file_path, 'r') as f:
source = f.read()
parser = GnuCParser()
ast = parser.parse(source)
# ast.show()
# generator = FunctionDeclarationGenerator()
generator = MyCGenerator()
source = generator.visit(ast)
# print(source)
print('\nStructs:')
for name in generator.structs:
if name and name.startswith('fz_'):
print(name)
print('\nFunctions:')
for name in generator.functions:
