def generate_org_fw_functions(src_files, org_functions_file,
org_functions_header, org_functions_lcs, symtab,
base_elf, gen_dir):
"""
determines the helper functions required to generate and writes them
:param src_files: list of source code files
:param org_functions_file: destination file for generated c-code
:param org_functions_header: destination file for generated header code
:param org_functions_lcs: destination file for generated linker symbol definitions
:param symtab: symbol table
:param base_elf: base firmware ELF file
:param gen_dir: destination directory for generated (intermediate) files
"""
# determine required original function helpers
req_org_funcs = Set()
for src_file in src_files:
text = preprocess_file(src_file, 'hexagon-cpp',
r'-DFW_WRAPPER="/dev/null"')
try: # pycparserext >= 2016.2
parser = GnuCParser(taboutputdir=gen_dir)
except TypeError:
parser = GnuCParser()
ast = parser.parse(text, src_file)
v = FuncCallVisitor()
v.visit(ast)
req_org_funcs = req_org_funcs.union(v.get_req_org_funcs())
# generate and write the functions
write_functions(req_org_funcs, org_functions_file, org_functions_header,
org_functions_lcs, symtab, base_elf)
def parse_c(filename, header_dir, use_fakes=False):
global __opts
if use_fakes:
fake_include_dir = fake_headers()
if fake_include_dir:
__opts.append('-I%s' % fake_include_dir)
parse_error_mo = re.compile(r'([/\w\.\-]+):(\d+):(\d+): before: .*')
ast = None
retry_parse = True
while retry_parse:
source = __preprocess(filename, header_dir)
parser = CParser()
try:
ast = parser.parse(source, filename)
retry_parse = False
except ParseError as e:
m = parse_error_mo.match(e.args[0])
if not m:
raise
opt = __guess_symbol(m.group(1), int(m.group(2)), int(m.group(3)))
if opt and opt not in __opts:
__opts.append(opt)
else:
raise
return ast
def from_code(self, code: str, parser=None):
"""
Takes (partial) C code as a string and returns the AST node that belongs to it. Wraps everything in a block.
Note: This function may not understand all code strings and shall be used with care. But, it understands
everything that is either directly valid C code, or partial code that can be put into a function body or as a
condition block (e.g. if and while conditions).
:param code: The C code.
:param parser: If you have already created a parser, you can hint it here. No need to create multiple parsers.
:return: The corresponding AST compound.
:rtype: c_ast.Compound or None
"""
# TODO this is a bit hacky, trying to wrap the code s.t. it is hopefully valid to allow for partial parsing.
if not parser: parser = GnuCParser()
try:
ast = parser.parse(code)
return c_ast.Compound(ast.ext)
except:
try:
wrapped_code = "void f() { " + code + " }"
ast = parser.parse(wrapped_code)
return ast.ext[0].body
except:
try:
wrapped_code = "void f() { if(" + code + ") {} }"
ast = parser.parse(wrapped_code)
return c_ast.Compound(
[ast.ext[0].body.block_items[0].cond])
except Exception as e:
print(e)
return None
def generate_patch_list(symtab, fw_wrapper, src_files, gen_dir):
"""
generates a list of patches needed to be applied in order to patch the ELF file
:param symtab: symbol table
:param fw_wrapper: firmware wrapper header file
:param src_files: list of patch source code files
:param gen_dir: destination directory for generated (intermediate) files
"""
patches = []
for filename in src_files:
text = preprocess_file(filename, 'hexagon-cpp',
'-DFW_WRAPPER="' + fw_wrapper + '"')
try: # pycparserext >= 2016.2
parser = GnuCParser(taboutputdir=gen_dir)
except TypeError:
parser = GnuCParser()
ast = parser.parse(text, filename)
v = FuncDefVisitor()
v.visit(ast)
patches.extend(v.patches)
return patches
def preprocess(path):
text = preprocess_file(path)
cparser = GnuCParser()
ast = cparser.parse(text, path)
generator = ext_c_generator.GnuCGenerator()
with open(path, "w") as o:
o.write(generator.visit(ast))
return ast
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_gnu_statement_expression():
src = """
int func(int a) {
return (int)({; ; *(int*)&a;});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_funky_header_code_5():
src = """ void do_foo(void) __asm(__STRING(do_foo));"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_gnu_statement_expression():
src = """
int func(int a) {
return (int)({; ; *(int*)&a;});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_funky_header_code_5():
src=""" void do_foo(void) __asm(__STRING(do_foo));"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_func_ret_ptr_decl_attribute():
src = """
extern void* memcpy(const void* src, const void *dst, int len) __attribute__((unused));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_nesty_c_declarator():
src = """
struct a {
int *b[1][1];
};
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_nesty_c_declarator():
src = """
struct a {
int *b[1][1];
};
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
def test_func_ret_ptr_decl_attribute():
src = """
extern void* memcpy(const void* src, const void *dst, int len) __attribute__((unused));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_ptr_decl_attribute():
src = """
int* __attribute__((weak)) array[256];
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_asm_volatile_1():
src = """
void read_tsc(void) {
long val;
asm("rdtsc" : "=A" (val));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_attributes():
src = """
int x[10] __attribute__((unused));
int y[20] __attribute((aligned(10)));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_array_attributes():
src = """
int x[10] __attribute__((unused));
int y[20] __attribute((aligned(10)));
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_asm_volatile_3():
src = """
void read_tsc(void) {
long fpenv;
asm("mtfsf 255,%0" :: "f" (fpenv));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_empty_struct_declaration():
src = """
typedef struct Foo {
} Foo_t;
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_empty_struct_declaration():
src = """
typedef struct Foo {
} Foo_t;
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_asm_volatile_3():
src = """
void read_tsc(void) {
long fpenv;
asm("mtfsf 255,%0" :: "f" (fpenv));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print GnuCGenerator().visit(ast)
def test_asm_volatile_2():
src = """
void read_tsc(void) {
long val;
asm volatile("rdtsc" : "=A" (val));
} """
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print GnuCGenerator().visit(ast)
def write_files(dir_list, header_include, out_dir, fn_names):
header_file = header_include.split('/')[-1]
is_valid_file = [
os.path.isfile(dir_name + header_file) for dir_name in dir_list
]
if not any(is_valid_file):
print "Skipped all functions in %s because it could not be found in %s." % (
header_file, dir_list)
return
else:
header_file = dir_list[is_valid_file.index(True)] + header_file
# pycparser utility function for preprocessing the file and
# getting all of the file as a string
text = preprocess_file(header_file)
# use pycparserext to parse the GNU C header file
p = GnuCParser()
ast = p.parse(text)
# loop over all functions requested
for fn in fn_names:
found = False
# loop over the ast
for node in ast.ext:
if isinstance(node, c_ast.FuncDef) or not isinstance(
node.type, c_ast.FuncDecl):
continue
# skip over functions with __noreturn__
if hasattr(node.type.type, 'attributes') and any([
e.name == "__noreturn__"
for e in node.type.type.attributes.exprs
]):
continue
if node.name != fn[0]:
continue
found = True
write_file(out_dir, header_include, node, fn[1], fn[2])
if not found:
print "The header file %s does not contain a method signature for %s" % (
header_file, fn[0])
def test_lvalue_gnu_statement_expression():
src = """
int func(int a) {
int ret=(int)({; ; *(int*)&a;});
return ret;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_func_decl_attribute():
src = """
extern void int happy(void) __attribute__((unused));
int main()
{
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_lvalue_gnu_statement_expression():
src = """
int func(int a) {
int ret=(int)({; ; *(int*)&a;});
return ret;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_func_decl_attribute():
src = """
extern void int happy(void) __attribute__((unused));
int main()
{
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_const_ptr_func_arg():
src = """
const int *bar;
void foo(const int *bar, int * const baz);
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
src_str = GnuCGenerator().visit(ast)
assert src_str.count("*") == 3
print(src_str)
def test_funky_header_code():
src = """
extern __inline int __attribute__ ((__nothrow__)) __signbitf (float __x)
{
int __m;
__asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
return __m & 0x8;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
from pycparserext.ext_c_generator import GnuCGenerator
print GnuCGenerator().visit(ast)
def test_const_ptr_func_arg():
src = """
const int *bar;
void foo(const int *bar, int * const baz);
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
src_str = GnuCGenerator().visit(ast)
assert src_str.count("*") == 3
print(src_str)
def enumerate_pins(c_source_file, include_dirs, definitions):
"""
Enumerate pins specified in PinNames.h, by looking for a PinName enum
typedef somewhere in the file.
"""
definitions += [
'__attribute(x)__=',
'__extension__(x)=',
'register=',
'__IO=',
'uint32_t=unsigned int',
'_UINT32_T_DECLARED'
]
gcc_args = ['-E', '-fmerge-all-constants']
gcc_args += ['-I' + directory for directory in include_dirs]
gcc_args += ['-D' + definition for definition in definitions]
parsed_ast = parse_file(c_source_file,
use_cpp=True,
cpp_path='arm-none-eabi-gcc',
cpp_args=gcc_args,
parser=GnuCParser())
# now, walk the AST
visitor = TypeDeclVisitor(['PinName'])
return visitor.visit(parsed_ast)
def test_empty_gnu_statement_expression():
# Incase, ASSERTS turn out to be empty statements
src = """
int func(int a) {
({
; ;
});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_funky_header_code():
src = """
extern __inline int __attribute__ ((__nothrow__)) __signbitf (float __x)
{
int __m;
__asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
return __m & 0x8;
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def test_empty_gnu_statement_expression():
# Incase, ASSERTS turn out to be empty statements
src = """
int func(int a) {
({
; ;
});
}
"""
from pycparserext.ext_c_parser import GnuCParser
p = GnuCParser()
ast = p.parse(src)
ast.show()
from pycparserext.ext_c_generator import GnuCGenerator
print(GnuCGenerator().visit(ast))
def generate_wrapper_lcs(filename, lcs_file, symtab_json_file, gen_dir):
"""
main linker script generation function
:param filename: firmware wrapper header
:param lcs_file: destination file
:param symtab_json_file: destination file for symbol table in JSON format
:param gen_dir: destination directory for generated (intermediate) files
"""
text = preprocess_file(filename, 'hexagon-cpp')
try: # pycparserext >= 2016.2
parser = GnuCParser(taboutputdir=gen_dir)
except TypeError:
parser = GnuCParser()
ast = parser.parse(text, filename)
v = DeclVisitor(lcs_file)
v.visit(ast)
json.dump(v.get_symtab(), open(symtab_json_file, 'w'))
def static_slicing_from_ast(ast: c_ast):
"""
Executes static slicing on the given AST.
:param ast: A PyCParser AST.
:return: A PyCParser AST on which static slicing has been applied.
:rtype: c_ast
"""
ast_file = tempfile.NamedTemporaryFile(delete=False, suffix=".c")
ast_file.write(bytes(GnuCGenerator().visit(ast), "utf-8"))
sliced_file = static_slicing_from_file(ast_file.name)
with open(sliced_file, "r") as slice:
return GnuCParser().parse(slice.read())
def extract_func_defs(filename):
# Note that cpp is used. Provide a path to your own cpp or
# make sure one exists in PATH.
ast = parse_file(filename,
use_cpp=True,
cpp_args=r'-I{}'.format(pycparser_fake_libc.directory),
parser=GnuCParser(lex_optimize=False,
yacc_optimize=False))
for node in ast.children():
decl = node[1]
if is_func(decl):
yield decl.name
def extract_signature(
func_name: str, body: str
) -> Optional[Tuple[str, List[Tuple[str, str]]]]:
signature = None
class FuncDefVisitor(c_ast.NodeVisitor):
def visit_FuncDeclExt(self, node):
nonlocal signature # bad
# so much badness below
def get_typ_str(node):
if isinstance(node, c_ast.PtrDecl):
return f"{' '.join(node.type.type.names)}*"
else:
return " ".join(node.type.names)
def get_arg_name(node):
if isinstance(node, c_ast.PtrDecl):
return node.type.declname
else:
return node.declname
if get_arg_name(node.children()[1][1]) == func_name:
return_typ = get_typ_str(node.children()[1][1])
params = node.children()[0][1]
signature = (
return_typ,
[(get_arg_name(p.type), get_typ_str(p.type)) for p in params]
if params
else [],
)
parser = GnuCParser()
ast = parser.parse(body)
v = FuncDefVisitor()
v.visit(ast)
# ast.show()
return signature
def _round_trip_matches(src):
from pycparserext.ext_c_parser import GnuCParser
from pycparserext.ext_c_generator import GnuCGenerator
p = GnuCParser()
first_ast = p.parse(src)
gen = GnuCGenerator().visit(first_ast)
second_ast = p.parse(gen)
if not _compare_asts(first_ast, second_ast):
print('First AST:')
first_ast.show()
print('Generated code:')
print(gen)
print('Second AST:')
second_ast.show()
return False
return True
def insert_k_into_induction_file(file_induction: str, k: int):
"""
Inserts the current unwind number into the file such that the property is only checked once the iteration number
has reached k.
:param file_induction: The file the verifier run on in the previous iteration.
:param k: The new k.
:return: A filename whose content is the C-code for the updated iteration number k.
"""
if k > 1:
# Parses the file (again) and identifies the main function (again). Not super elegant, could surely be improved.
with open(file_induction) as file:
ast = GnuCParser().parse(file.read())
main_function = CAnalyzer(ast).identify_function(MAIN_FUNCTION_NAME)
CTransformer(ast).replace_initial_value(
"const unsigned int k = " + str(k - 1) + ";", main_function, k)
output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".c")
output_file.write(bytes(GnuCGenerator().visit(ast), "utf-8"))
return output_file.name
else:
return file_induction
def static_slicing_from_file(input_file: str,
output: str = None,
slice_funcs: list = None):
"""
Executes static slicing from a given file name. Writes to the output file if given, otherwise creates a temporary
file and writes the results to it.
:param input_file: The location of the input file.
:param output: An optional file location.
:param slice_funcs: The functions to slice for their reachability. If None, the default is __VERIFIER_error.
:return: The file name of the output file.
:rtype: str
"""
# Need a temporary file because Frama-C does not allow to only print out the C code.
slice_file = tempfile.NamedTemporaryFile(delete=False, suffix=".c")
if not slice_funcs or len(slice_funcs) == 0:
slice_funcs = ["__VERIFIER_error"]
with open(input_file, "r") as input:
# Only keep assume statements if there are any, otherwise Frama-C complains.
if "__VERIFIER_assume" in input.read().replace(
"extern int __VERIFIER_assume(int);", ""):
slice_funcs.append("__VERIFIER_assume")
# /dev/null supresses output of Frama-C.
with open(os.devnull, "w") as devnull:
subprocess.call([
"frama-c", "-slice-calls", ",".join(slice_funcs), input_file,
"-then-on", "Slicing export", "-print", "-ocode", slice_file.name
],
stdout=devnull)
slice = slice_file.read().decode("utf-8")
# Removes comments of the form /**/ as GnuCParser does not seem to like them.
slice = re.sub("/\*.*?\*/", "", slice)
# Parses the slice and original to allow for the process of moving switch-local variables out of switch blocks.
slice_ast = GnuCParser().parse(slice)
# By moving switch-local variables outside of the switch-statement, we prevent a bug in CBMC v5.11 and earlier.
move_switch_local_variables(slice_ast)
# Output printing.
if not output:
output_file = tempfile.NamedTemporaryFile(delete=False, suffix=".c")
output = open(output_file.name, "w")
output.write(GnuCGenerator().visit(slice_ast))
return output.name
def variable_analysis_from_file(input_file: str,
output: str = None,
ignore_functions=None):
"""
Executes the variable analysis from a given file name. Writes to the output file if given, otherwise creates a
temporary file and writes the results to it.
:param input_file: The location of the input file.
:param output: An optional file location.
:param ignore_functions: A set of functions whose contents are ignored when determining the usage of variables. If
None, all functions are taken into account.
:return: The file name of the output file.
:rtype: str
"""
with open(input_file) as input:
ast = GnuCParser().parse(input.read())
changed_ast = variable_analysis_from_ast(ast, ignore_functions)
if not output:
output_file = tempfile.NamedTemporaryFile(delete=False,
suffix=".c")
output = open(output_file.name, "w")
output.write(GnuCGenerator().visit(changed_ast))
return output.name
def visit_AttributeSpecifier(self, n):
return ' __attribute__((' + self.visit(n.exprlist) + '))'
####################################################################################################
class MyCGenerator(AsmAndAttributesMixin, CGenerator):
pass
####################################################################################################
file_path = 'fitz-from-cpp.h'
with open(file_path, 'r') as f:
source = f.read()
parser = GnuCParser()
ast = parser.parse(source)
# ast.show()
# generator = FunctionDeclarationGenerator()
generator = MyCGenerator()
source = generator.visit(ast)
# print(source)
print('\nStructs:')
for name in generator.structs:
if name and name.startswith('fz_'):
print(name)
print('\nFunctions:')
from pycparserext.ext_c_parser import GnuCParser p = GnuCParser() src=""" typedef int wchar_t; extern wchar_t *wcscpy (wchar_t *restrict __dest, const wchar_t *__restrict __src) __attribute ((nothrow , leaf)); """ p.parse(src).show()
