def run_example_tool(bat_file):
"""
This method is called from the run_analysis method. It is called for
each matching file. Files are matched against the glob expression
specified in main.
When this method is called, the script will have changed to the directory
where the batch file exists.
"""
# In order to run a source code analysis tool, build appropriate command
# line(s) as shown in the commented out example below
"""
build_name = "toolname.c_and_cpp." + py_common.get_timestamp() + "." + bat_file[:-4]
command1 = "mytool --build " + build_name + " --option1 --option2 " + bat_file
py_common.print_with_timestamp("Running " + command1)
py_common.run_commands([command1])
command2 = "mytool --analyze " + build_name + " --output " + build_name + ".xml"
py_common.print_with_timestamp("Running " + command2)
py_common.run_commands([command2])
"""
# The code below will just run the batch file to compile the test cases without using a tool
# Remove or comment out this code when modifying this file to use an analysis tool
command = bat_file
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command])
def run_example_tool(bat_file):
"""
This method is called from the run_analysis method. It is called for
each matching file. Files are matched against the glob expression
specified in main.
When this method is called, the script will have changed to the directory
where the batch file exists.
"""
# In order to run a source code analysis tool, build appropriate command
# line(s) as shown in the commented out example below
"""
build_name = "toolname.c_and_cpp." + py_common.get_timestamp() + "." + bat_file[:-4]
command1 = "mytool --build " + build_name + " --option1 --option2 " + bat_file
py_common.print_with_timestamp("Running " + command1)
py_common.run_commands([command1])
command2 = "mytool --analyze " + build_name + " --output " + build_name + ".xml"
py_common.print_with_timestamp("Running " + command2)
py_common.run_commands([command2])
"""
# The code below will just run the batch file to compile the test cases without using a tool
# Remove or comment out this code when modifying this file to use an analysis tool
command = bat_file
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command])
def generate_calls_to_fxs(testcase_files):
fcl = FunctionCallLines()
for fullfilepath in testcase_files:
filename = os.path.basename(fullfilepath)
# do different things if its a c or cpp file
match = re.search(
"^(?P<root>CWE(\d+).*__.*_\d+)((?P<letter>[a-z]*)|_(bad|good\d+))(\.c|\.cpp)$",
filename)
if filename.endswith(".cpp"):
root = match.group(
"root") # we don't use the letter in the namespace
bad = "bad();"
good = "good();"
fcl.cpp_bad_lines.append("\tprintLine(\"Calling " + root + "::" +
bad + "\");")
fcl.cpp_bad_lines.append("\t" + root + "::" + bad + "\n")
fcl.cpp_h_bad_lines.append("\tnamespace " + root + " { void " +
bad + "}\n")
fcl.cpp_good_lines.append("\tprintLine(\"Calling " + root + "::" +
good + "\");")
fcl.cpp_good_lines.append("\t" + root + "::" + good + "\n")
fcl.cpp_h_good_lines.append("\tnamespace " + root + " { void " +
good + "}\n")
elif filename.endswith(".c"):
# we only want to add the "a" files
if match.group("letter") != "" and match.group("letter") != "a":
py_common.print_with_timestamp("Ignored file: " + filename)
continue
root = match.group("root")
bad = "_bad();"
good = "_good();"
fcl.c_bad_lines.append("\tprintLine(\"Calling " + root + bad +
"\");")
fcl.c_bad_lines.append("\t" + root + bad + "\n")
fcl.c_h_bad_lines.append("\tvoid " + root + bad + "\n")
# don't create good if template file contains point-flaw-badonly.
file_contents = py_common.open_file_and_get_contents(fullfilepath)
result = re.search("Template File: point-flaw-badonly",
file_contents, re.IGNORECASE)
if result == None:
fcl.c_good_lines.append("\tprintLine(\"Calling " + root +
good + "\");")
fcl.c_good_lines.append("\t" + root + good + "\n")
fcl.c_h_good_lines.append("\tvoid " + root + good + "\n")
else:
raise Exception("filename ends with something we don't handle!: " +
fullfilepath)
return fcl
def generate_calls_to_linux_fxs(testcase_files):
fcl = FunctionCallLines()
for fullfilepath in testcase_files:
filename = os.path.basename(fullfilepath)
# we only want the non w32 and non wchar_t test cases
if ('w32' not in filename) and ('wchar_t' not in filename):
# do different things if its a c or cpp file
match = re.search("^(?P<root>CWE(\d+).*__.*_\d+)((?P<letter>[a-z]*)|_(bad|good\d+))(\.c|\.cpp)$", filename)
if filename.endswith(".cpp"):
root = match.group("root") # we don't use the letter in the namespace
bad = "bad();"
good = "good();"
fcl.cpp_bad_lines.append("\tprintLine(\"Calling " + root + "::" + bad + "\");");
fcl.cpp_bad_lines.append("\t" + root + "::" + bad + "\n")
fcl.cpp_h_bad_lines.append("\tnamespace " + root + " { void " + bad + "}\n")
fcl.cpp_good_lines.append("\tprintLine(\"Calling " + root + "::" + good + "\");");
fcl.cpp_good_lines.append("\t" + root + "::" + good + "\n")
fcl.cpp_h_good_lines.append("\tnamespace " + root + " { void " + good + "}\n")
elif filename.endswith(".c"):
# we only want to add the "a" files
if match.group("letter") != "" and match.group("letter") != "a":
py_common.print_with_timestamp("Ignored file: " + filename)
continue
root = match.group("root")
bad = "_bad();"
good = "_good();"
fcl.c_bad_lines.append("\tprintLine(\"Calling " + root + bad + "\");");
fcl.c_bad_lines.append("\t" + root + bad + "\n")
fcl.c_h_bad_lines.append("\tvoid " + root + bad + "\n")
# don't create good if template file contains point-flaw-badonly.
file_contents = py_common.open_file_and_get_contents(fullfilepath)
result = re.search("Template File: point-flaw-badonly", file_contents, re.IGNORECASE)
if result == None:
fcl.c_good_lines.append("\tprintLine(\"Calling " + root + good + "\");");
fcl.c_good_lines.append("\t" + root + good + "\n")
fcl.c_h_good_lines.append("\tvoid " + root + good + "\n")
else:
raise Exception("filename ends with something we don't handle!: " + fullfilepath)
return fcl
def run_example_tool(bat_file):
"""
This method is called from the run_analysis method. It is called for
each matching file. Files are matched against the glob expression
specified in main.
When this method is called, the script will have changed to the directory
where the batch file exists.
"""
# In order to run a source code analysis tool, build appropriate command
# line(s) as shown in the commented out example below
"""
build_name = "toolname.csharp." + py_common.get_timestamp() + "." + bat_file[:-4]
command1 = "mytool --build " + build_name + " --option1 --option2 " + bat_file
py_common.print_with_timestamp("Running " + command1)
py_common.run_commands([command1])
command2 = "mytool --analyze " + build_name + " --output " + build_name + ".xml"
py_common.print_with_timestamp("Running " + command2)
py_common.run_commands([command2])
"""
msbuildPath = "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current\\Bin\\MSBuild.exe"
solutionPath = "D:\\GitHub\\MyGit\\JulietTestSuite\\Example\\Example.csproj"
parameter = "-p:Configuration=Release"
command1 = msbuildPath + " " + solutionPath + " " + parameter
py_common.print_with_timestamp("Running " + command1)
py_common.run_commands([command1])
# The code below will just run ant to compile the test cases without using a
# tool
# Remove or comment out this code when modifying this file to use an analysis
# tool
command = bat_file
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command])
def create_xml_dir(self):
# create, or empty, 'xmls' folder
#
# Note: Deleting entire folder and then re-creating it immediately sometimes conflicts
# with anti-virus sortware and cannot always release handles quick enough, so the entire
# parent folder is not deleted, only the files withing it. This prevents this problem
#
if not os.path.exists(self.dest_path):
py_common.print_with_timestamp("The path \"" + self.dest_path +
"\" does not exist")
py_common.print_with_timestamp("creating directory \"" +
self.dest_path + "\"")
os.makedirs(self.dest_path)
else:
py_common.print_with_timestamp(
self.dest_path + " already exists. Cleaning before use...")
fileList = os.listdir(self.dest_path)
for fileName in fileList:
# os.remove(self.dest_path + "//" + fileName)
os.remove(os.path.join(self.dest_path, fileName))
# fortify files are not in standard xml format
if self.tool_name == 'fortify':
self.scan_data_files = py_common.find_files_in_dir(
self.source_path, '.*?\.fpr$')
else:
self.scan_data_files = py_common.find_files_in_dir(
self.source_path, '.*?\.xml$')
def run_example_tool(build_xml_file):
"""
This method is called from the run_analysis method. It is called for
each matching file. Files are matched against the glob expression
specified in main.
When this method is called, the script will have changed to the directory
where the build.xml file exists.
"""
# In order to run a source code analysis tool, build appropriate command
# line(s) as shown in the commented out example below
"""
# retrieve the CWE # from the parent directory name
path = os.getcwd()
cwe_id = re.search("(CWE\d+)_", os.path.basename(path)).group(1)
build_name = "toolname.java." + py_common.get_timestamp() + "." + cwe_id
command1 = "mytool --build " + build_name + " --option1 --option2 " + build_xml_file
py_common.print_with_timestamp("Running " + command1)
py_common.run_commands([command1], True)
command2 = "mytool --analyze " + build_name + " --output " + build_name + ".xml"
py_common.print_with_timestamp("Running " + command2)
py_common.run_commands([command2], True)
"""
# The code below will just run ant to compile the test cases without using a tool
# Remove or comment out this code when modifying this file to use an analysis tool
command = "ant"
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command], True)
def run_example_tool(build_xml_file):
"""
This method is called from the run_analysis method. It is called for
each matching file. Files are matched against the glob expression
specified in main.
When this method is called, the script will have changed to the directory
where the build.xml file exists.
"""
# In order to run a source code analysis tool, build appropriate command
# line(s) as shown in the commented out example below
"""
# retrieve the CWE # from the parent directory name
path = os.getcwd()
cwe_id = re.search("(CWE\d+)_", os.path.basename(path)).group(1)
build_name = "toolname.java." + py_common.get_timestamp() + "." + cwe_id
command1 = "mytool --build " + build_name + " --option1 --option2 " + build_xml_file
py_common.print_with_timestamp("Running " + command1)
py_common.run_commands([command1], True)
command2 = "mytool --analyze " + build_name + " --output " + build_name + ".xml"
py_common.print_with_timestamp("Running " + command2)
py_common.run_commands([command2], True)
"""
# The code below will just run ant to compile the test cases without using a tool
# Remove or comment out this code when modifying this file to use an analysis tool
command = "ant"
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command], True)
def run_fortify_c_cpp(bat_file):
"""
Build and analyze the source code using the batch file.
"""
# build_name is based upon the name of the batch file
build_name = get_build_name(bat_file)
build_id = TOOL_NAME.replace(
" ", "_") #Replace any spaces in the tool name with underscore
build_id += "." + project_prefix
build_id += "." + py_common.get_timestamp()
build_id += "." + build_name
# Create file names and paths - we do this here so that the commands
# generated below can remain unchanged as long as there are no new options
# being passed to Fortify
build_log_filename = build_id + "-build-log.txt"
scan_log_filename = build_id + "-scan-log.txt"
clean_log_filename = build_id + "-clean-log.txt"
fpr_file = os.path.join(output_path, build_id) + ".fpr"
# Build the command to compile the code
command = MAIN_TOOL_COMMAND
command += " " + "-b" + " " + build_id
command += " " + "-logfile" + " " + build_log_filename
command += " " + "touchless"
command += " " + bat_file
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command])
# Build the command to analyze the code
command = MAIN_TOOL_COMMAND
command += " " + "-b" + " " + build_id
command += " " + "-logfile" + " " + scan_log_filename
command += " " + "-scan"
command += " " + "-f" + " \"" + fpr_file + "\""
command += " " + "-Dcom.fortify.sca.limiters.MaxIndirectResolutionsForCall=" + MAX_INDIRECT_RESOLUTIONS_FOR_CALL
command += " " + "-Dcom.fortify.sca.limiters.MaxFunPtrsForCall=" + MAX_FUN_PTRS_FOR_CALL
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command])
# Perform a clean so that we don't fill up the HD
command = MAIN_TOOL_COMMAND
command += " " + "-b" + " " + build_id
command += " " + "-logfile" + " " + clean_log_filename
command += " " + "-clean"
py_common.print_with_timestamp("Running " + command)
py_common.run_commands([command])
def make_stats():
with open('tests.json', 'r') as f:
tests = json.load(f)
py_common.print_with_timestamp("Stats on " + str(len(tests)) +
" tests ...")
test_results = {}
for test in tests:
"""
Return test description which is a simple parsing of the test file path
bacause it contains all the information we need.
"""
cwe, directory, filename, test_name = get_test_info(test)
"""
Explore the output dir generated by Klee and retrieve test results
"""
compilation_issue, detected, undetected = retrieveTestResult(test_name)
if cwe in test_results.keys():
test_results[cwe]["compilation_issue"] = test_results[cwe][
"compilation_issue"] + compilation_issue
test_results[cwe][
"undetected"] = test_results[cwe]["undetected"] + undetected
test_results[cwe][
"detected"] = test_results[cwe]["detected"] + detected
else:
test_results[cwe] = {
"total": 1,
"compilation_issue": compilation_issue,
"undetected": undetected,
"detected": detected
}
py_common.print_with_timestamp("Testing " + test["name"])
print(tabulate())
py_common.print_with_timestamp("Done ")
sys.path.append("..")
import py_common
import update_Main_java_ServletMain_java_and_web_xml
def help():
sys.stderr.write('Usage: \n')
sys.stderr.write(' create_per_cwe_files.py (builds per CWE files for all testcases)\n')
sys.stderr.write(' create_per_cwe_files.py CWE (builds per CWE files for all testcases)\n')
sys.stderr.write(' create_per_cwe_files.py CWE(78|15) (builds per CWE files for test cases for CWE 78 and CWE 15)')
if __name__ == "__main__":
# check if ./src/testcases directory exists, if not, we are running
# from wrong working directory
if not os.path.exists("src\\testcases"):
py_common.print_with_timestamp("Wrong working directory; could not find testcases directory")
exit()
# default value which is used if no arguments are passed on command line
cwe_regex="CWE"
if len(sys.argv) > 2:
help()
exit()
if len(sys.argv) == 2:
if (sys.argv[1] == '-h'):
help()
exit()
cwe_regex = sys.argv[1]
# may need /bigobj flag: http://msdn.microsoft.com/en-us/library/ms173499%28VS.90%29.aspx
# Only one of our C/C++ tools requires debug flags so the debug flags that are set are specific for this tool
debug_flags = '/I"..\\..\\testcasesupport" /Zi /Od /MTd /GS- /INCREMENTAL:NO /DEBUG /W3 /bigobj /EHsc /nologo' # if this line is modified, change the one below
split_debug_flags = '/I"..\\..\\..\\testcasesupport" /Zi /Od /MTd /GS- /INCREMENTAL:NO /DEBUG /W3 /bigobj /EHsc /nologo'
linker_flags = '/I"..\\..\\testcasesupport" /W3 /MT /GS /RTC1 /bigobj /EHsc /nologo' # if this line is modified, change the one below
split_linker_flags = '/I"..\\..\\..\\testcasesupport" /W3 /MT /GS /RTC1 /bigobj /EHsc /nologo'
compile_flags = linker_flags + " /c"
split_compile_flags = split_linker_flags + " /c"
debug_compile_flags = debug_flags + " /c"
split_debug_compile_flags = split_debug_flags + " /c"
if __name__ == "__main__":
# check if ./testcases directory exists, if not, we are running
# from wrong working directory
if not os.path.exists("testcases"):
py_common.print_with_timestamp("Wrong working directory; could not find testcases directory")
exit()
# default values which are used if no arguments are passed on command line
cwe_regex = "CWE"
use_debug = False
if len(sys.argv) > 1:
if ((sys.argv[1] == '-h') or (len(sys.argv) > 3)):
help()
exit()
if len(sys.argv) == 2:
cwe_regex = sys.argv[1]
def update_csharp_templates(testcase_location=".",
main_path="src\\testcasesupport\\"):
"""
Update Program.cs
"""
# get list of testcase files
testcase_files = build_list_of_csharp_testcase_files(testcase_location)
# Get the TestCaseSupport.csproj GUID
testcasesupport_csproj = "src\\TestCaseSupport\\TestCaseSupport\\TestCaseSupport.csproj"
tree = ET.parse(testcasesupport_csproj)
root = tree.getroot()
ns = {'pg': 'http://schemas.microsoft.com/developer/msbuild/2003'}
property_groups = root.findall('pg:PropertyGroup', ns)
tcs_project_guid = property_groups[0].find('pg:ProjectGuid', ns)
if (len(testcase_files) > 0):
# build up the class instantiation lines
testcase_lines = []
namespace = ""
for fullfilepath in testcase_files:
filename = os.path.basename(fullfilepath)
# figure out the namespace based on the file path
tc_index = fullfilepath.index(os.path.join("src", "testcases"))
f_index = fullfilepath.index(filename)
namespace = "testcases" + fullfilepath[tc_index + len(
os.path.join("src", "testcases")):f_index].replace("\\", ".")
# remove .cs from the end of the filename to get the classname
classname = filename[:-3]
if "Web" not in classname:
line = "\t\t\t(new " + classname + \
"()).RunTest(\"" + classname + "\");"
testcase_lines.append(line)
# Remove the trailing '.' from the namespace
namespace = namespace[:-1]
# Check if this is a split directory and remove the 's##' if so
r1 = re.compile('\.s\d\d$')
if r1.search(namespace):
namespace = namespace[:-4]
# update Program_cs
update_Program(main_path, testcase_lines, namespace)
cwe_and_name = os.path.basename(os.path.dirname(testcase_files[0]))
if cwe_and_name.startswith('s'):
cwe_and_name = os.path.basename(
os.path.dirname(os.path.dirname(testcase_files[0])))
csproj_guid = build_guid(main_path + ".csproj")
# update the VS Solution template and rename it to the CWE name
update_and_rename_Main_sln(main_path, cwe_and_name, csproj_guid,
tcs_project_guid.text)
# update the VS CS Project template and rename it to the CWE name
update_and_rename_Main_csproj(main_path, cwe_and_name, csproj_guid,
tcs_project_guid.text)
# Update the AssemblyInfo.cs template
update_AssemblyInfo_cs(main_path, cwe_and_name, csproj_guid)
else:
py_common.print_with_timestamp("[INFO] Skipping directory: " +
testcase_location)
def run_example_tool(test_core):
FNULL = open(os.devnull, 'w')
for test in tests:
FLAGS = [
"-D" + test["name"],
"-I/home/noname/Inception2/Inception-analyzer/include"
]
directory = os.path.dirname(test["filepath"])
filename = os.path.basename(test["filepath"])
py_common.print_with_timestamp("Testing " + test["name"])
output_dir = "/home/noname/Inception2/Juliet_test_suite/results/" + test[
"name"]
test_file_src = directory + "/" + test["name"] + ".ll"
test_file_ll = test["name"] + ".ll"
test_file_bc = test["name"] + ".bc"
"""
Klee will not run if the output directory is not empty
"""
dir = Path(output_dir)
if dir.is_dir():
continue
"""
Check if we need to compile or not
"""
dir = Path("tests_klee/" + test_file_bc)
if dir.is_file():
try:
# P = subprocess.check_output(["klee", "-max-time=30", "-output-dir=/media/noname/3a224af4-22de-4deb-ad88-08422268a9fc/Inception/Benchmark/klee"+test["name"], test_file_bc], timeout=10)
p = subprocess.Popen([
"klee", "-max-time=300", "-output-dir=" + output_dir,
test_file_bc
],
stderr=FNULL,
stdout=FNULL)
p.wait(timeout=300)
# py_common.run_commands(["klee", "-output-dir=/media/noname/3a224af4-22de-4deb-ad88-08422268a9fc/Inception/Benchmark/klee"+test["name"], test_file_bc], use_shell=True, stdout=False, stderr=False)
except CalledProcessError:
colorlog.error(
'One test failed during execution in Klee. Test named ' +
test["name"])
sys.stdout.write("\033[K")
continue
except subprocess.TimeoutExpired:
p.kill()
sys.stdout.write("\033[K")
continue
continue
try:
#Compile vuln.c with tested function activated
if filename.endswith(".c"):
py_common.run_commands(CC + CFLAGS + INCLUDES + FLAGS +
["testcasesupport/main_linux.cpp"] +
[test["filepath"]])
elif filename.endswith(".cpp"):
py_common.run_commands(CPP + CXXFLAGS + INCLUDES + FLAGS +
["testcasesupport/main_linux.cpp"] +
[test["filepath"]])
else:
print("Unsupported file extension" + file_name)
except CalledProcessError:
# colorlog.error('Aborted, press any key to continue')
print("Aborted test due to compilation issue")
# key = input()
continue
py_common.run_commands([
"llvm-link", test_file_ll, test_core, "./main_linux.ll", "-o",
test_file_bc
])
py_common.run_commands(
["mv", test_file_ll, "tests_klee/" + test_file_ll],
use_shell=False)
# output_dir = "/media/noname/3a224af4-22de-4deb-ad88-08422268a9fc/Inception/Benchmark/JULIA/klee"+test["name"]
try:
# P = subprocess.check_output(["klee", "-max-time=30", "-output-dir=/media/noname/3a224af4-22de-4deb-ad88-08422268a9fc/Inception/Benchmark/klee"+test["name"], test_file_bc], timeout=10)
p = subprocess.Popen([
"klee", "-max-time=300", "-output-dir=" + output_dir,
test_file_bc
],
stderr=FNULL,
stdout=FNULL)
p.wait(timeout=300)
# py_common.run_commands(["klee", "-output-dir=/media/noname/3a224af4-22de-4deb-ad88-08422268a9fc/Inception/Benchmark/klee"+test["name"], test_file_bc], use_shell=True, stdout=False, stderr=False)
except CalledProcessError:
colorlog.error(
'One test failed during execution in Klee. Test named ' +
test["name"])
sys.stdout.write("\033[K")
continue
except subprocess.TimeoutExpired:
p.kill()
sys.stdout.write("\033[K")
continue
py_common.print_with_timestamp("Done ")
FNULL.close()
# may need /bigobj flag: http://msdn.microsoft.com/en-us/library/ms173499%28VS.90%29.aspx
# Only one of our C/C++ tools requires debug flags so the debug flags that are set are specific for this tool
debug_flags = '/I"..\\..\\testcasesupport" /Zi /Od /MTd /GS- /INCREMENTAL:NO /DEBUG /W3 /bigobj /EHsc /nologo' # if this line is modified, change the one below
split_debug_flags = '/I"..\\..\\..\\testcasesupport" /Zi /Od /MTd /GS- /INCREMENTAL:NO /DEBUG /W3 /bigobj /EHsc /nologo'
linker_flags = '/I"..\\..\\testcasesupport" /W3 /MT /GS /RTC1 /bigobj /EHsc /nologo' # if this line is modified, change the one below
split_linker_flags = '/I"..\\..\\..\\testcasesupport" /W3 /MT /GS /RTC1 /bigobj /EHsc /nologo'
compile_flags = linker_flags + " /c"
split_compile_flags = split_linker_flags + " /c"
debug_compile_flags = debug_flags + " /c"
split_debug_compile_flags = split_debug_flags + " /c"
if __name__ == "__main__":
# check if ./testcases directory exists, if not, we are running
# from wrong working directory
if not os.path.exists("testcases"):
py_common.print_with_timestamp(
"Wrong working directory; could not find testcases directory")
exit()
# default values which are used if no arguments are passed on command line
cwe_regex = "CWE"
use_debug = False
if len(sys.argv) > 1:
if ((sys.argv[1] == '-h') or (len(sys.argv) > 3)):
help()
exit()
if len(sys.argv) == 2:
cwe_regex = sys.argv[1]
files = py_common.find_files_in_dir(directory, testregex)
dirs = set()
for file in files:
base_dir = os.path.dirname(file)
dirs.add(base_dir)
return dirs
if __name__ == "__main__":
# check if ./testcases directory exists, if not, we are running
# from wrong working directory
if not os.path.exists("testcases"):
py_common.print_with_timestamp(
"Wrong working directory; could not find testcases directory")
exit()
global testregex
# Only include the tests cases that are relevant, that don't involve sockets
#testregex = "CWE(122|XXX|YYY|ZZZ)+((?!socket).)*(\.c)$"
testregex = "CWE(665)+((?!socket).)*(\.c)$"
# update main_linux.cpp/testcases.h
# Only take the test cases we care about, and only the .c tests
testcase_files = asan_update_main_cpp_and_testcases_h.build_list_of_primary_c_cpp_testcase_files(
"testcases", [testregex])
fcl = asan_update_main_cpp_and_testcases_h.generate_calls_to_fxs(
testcase_files)
linux_fcl = asan_update_main_cpp_and_testcases_h.generate_calls_to_linux_fxs(
testcase_files)
def generate_calls_to_linux_fxs(testcase_files):
fcl = FunctionCallLines()
for fullfilepath in testcase_files:
filename = os.path.basename(fullfilepath)
filename_no_extension = os.path.splitext(filename)[0]
# we only want the non w32 and non wchar_t test cases
if ('w32' not in filename) and ('wchar_t' not in filename) and ('fgets' not in filename) and ('socket' not in filename):
# do different things if its a c or cpp file
match = re.search("^(?P<root>CWE(\d+).*__.*_\d+)((?P<letter>[a-z]*)|_(bad|good\d+))(\.c|\.cpp)$", filename)
if filename.endswith(".cpp"):
root = match.group("root") # we don't use the letter in the namespace
bad = "bad();"
# good = "good();"
# fcl.cpp_bad_lines.append("\tprintLine(\"Calling " + root + "::" + bad + "\");");
fcl.cpp_bad_lines.append("#ifdef " + filename_no_extension)
fcl.cpp_bad_lines.append("\t" + root + "::" + bad + "\n")
fcl.cpp_bad_lines.append("#endif")
fcl.cpp_h_bad_lines.append("#ifdef " + filename_no_extension)
fcl.cpp_h_bad_lines.append("\tnamespace " + root + " { void " + bad + "}\n")
fcl.cpp_h_bad_lines.append("#endif")
fcl.tests.append({"filepath": fullfilepath, "name": filename_no_extension, "type": "CPP"})
# fcl.cpp_good_lines.append("\tprintLine(\"Calling " + root + "::" + good + "\");");
# fcl.cpp_good_lines.append("\t" + root + "::" + good + "\n")
# fcl.cpp_h_good_lines.append("\tnamespace " + root + " { void " + good + "}\n")
elif filename.endswith(".c"):
# we only want to add the "a" files
if match.group("letter") != "" and match.group("letter") != "a":
py_common.print_with_timestamp("Ignored file: " + filename)
continue
root = match.group("root")
bad = "_bad();"
# good = "_good();"
# fcl.c_bad_lines.append("\tprintLine(\"Calling " + root + bad + "\");");
fcl.c_bad_lines.append("#ifdef " + filename_no_extension)
fcl.c_bad_lines.append("\t" + root + bad + "\n")
fcl.c_bad_lines.append("#endif")
fcl.c_h_bad_lines.append("#ifdef " + filename_no_extension)
fcl.c_h_bad_lines.append("\tvoid " + root + bad + "\n")
fcl.c_h_bad_lines.append("#endif")
fcl.tests.append({"filepath": fullfilepath, "name": filename_no_extension, "type": "C"})
# don't create good if template file contains point-flaw-badonly.
# file_contents = py_common.open_file_and_get_contents(fullfilepath)
# result = re.search("Template File: point-flaw-badonly", file_contents, re.IGNORECASE)
# if result == None:
# fcl.c_good_lines.append("\tprintLine(\"Calling " + root + good + "\");");
# fcl.c_good_lines.append("\t" + root + good + "\n")
# fcl.c_h_good_lines.append("\tvoid " + root + good + "\n")
else:
raise Exception("filename ends with something we don't handle!: " + fullfilepath)
with open('tests.json', mode='wt', encoding='utf-8') as myfile:
json.dump(fcl.tests, myfile, indent=2)
return fcl
def generate_calls_to_linux_fxs(testcase_files):
fcl = FunctionCallLines()
for fullfilepath in testcase_files:
filename = os.path.basename(fullfilepath)
# we only want the non w32 and non wchar_t test cases
if ('w32' not in filename) and ('wchar_t' not in filename):
# do different things if its a c or cpp file
match = re.search(
"^(?P<root>CWE(\d+).*__.*_\d+)((?P<letter>[a-z]*)|_(bad|good\d+))(\.c|\.cpp)$",
filename)
if filename.endswith(".cpp"):
root = match.group(
"root") # we don't use the letter in the namespace
bad = "bad();"
good = "good();"
fcl.cpp_bad_lines.append("\tprintLine(\"Calling " + root +
"::" + bad + "\");")
fcl.cpp_bad_lines.append("\t" + root + "::" + bad + "\n")
fcl.cpp_h_bad_lines.append("\tnamespace " + root + " { void " +
bad + "}\n")
fcl.cpp_good_lines.append("\tprintLine(\"Calling " + root +
"::" + good + "\");")
fcl.cpp_good_lines.append("\t" + root + "::" + good + "\n")
fcl.cpp_h_good_lines.append("\tnamespace " + root +
" { void " + good + "}\n")
elif filename.endswith(".c"):
# we only want to add the "a" files
if match.group("letter") != "" and match.group(
"letter") != "a":
py_common.print_with_timestamp("Ignored file: " + filename)
continue
root = match.group("root")
bad = "_bad();"
good = "_good();"
fcl.c_bad_lines.append("\tprintLine(\"Calling " + root + bad +
"\");")
fcl.c_bad_lines.append("\tchildproc = fork();")
fcl.c_bad_lines.append("\tif(childproc > 0) {")
fcl.c_bad_lines.append(
"\t\twaitpid(childproc, &returnCode, 0);")
fcl.c_bad_lines.append(
"\t\tif(WIFEXITED(returnCode) && (WEXITSTATUS(returnCode) == 1)) {"
)
fcl.c_bad_lines.append("\t\t\tprintf(\"" + root + bad +
" passed\\n\");")
fcl.c_bad_lines.append("\t\t\tpassedBadTests++;")
fcl.c_bad_lines.append(
"\t\t} else if(WIFSIGNALED(returnCode) && WTERMSIG(returnCode) == SIGALRM) {"
)
fcl.c_bad_lines.append("\t\t\tprintf(\"" + root + bad +
" timed out\\n\");")
fcl.c_bad_lines.append("\t\t\ttimedOutBadTests++;")
fcl.c_bad_lines.append("\t\t} else {")
fcl.c_bad_lines.append("\t\t\tprintf(\"" + root + bad +
" failed\\n\");")
fcl.c_bad_lines.append("\t\t\tfprintf(stderr, \"%s\\n\", \"" +
root + bad + "\");")
fcl.c_bad_lines.append("\t\t\tfailedBadTests++;")
fcl.c_bad_lines.append("\t\t}")
fcl.c_bad_lines.append("\t} else {")
fcl.c_bad_lines.append("\t\talarm(ALERT_TIMEOUT);")
fcl.c_bad_lines.append("\t\t" + root + bad + "\n\t\texit(0);")
fcl.c_bad_lines.append("\t}\n\ttotalBadTests++;\n")
fcl.c_h_bad_lines.append("\tvoid " + root + bad + "\n")
# don't create good if template file contains point-flaw-badonly.
file_contents = py_common.open_file_and_get_contents(
fullfilepath)
result = re.search("Template File: point-flaw-badonly",
file_contents, re.IGNORECASE)
if result == None:
fcl.c_good_lines.append("\tprintLine(\"Calling " + root +
good + "\");")
fcl.c_good_lines.append("\tchildproc = fork();")
fcl.c_good_lines.append("\tif(childproc > 0) {")
fcl.c_good_lines.append(
"\t\twaitpid(childproc, &returnCode, 0);")
fcl.c_good_lines.append(
"\t\tif(WIFEXITED(returnCode) && WEXITSTATUS(returnCode) == 1) {"
)
fcl.c_good_lines.append("\t\t\tprintf(\"" + root + good +
" failed\\n\");")
fcl.c_good_lines.append(
"\t\t\tfprintf(stderr, \"%s\\n\", \"" + root + good +
"\");")
fcl.c_good_lines.append("\t\t\tfailedGoodTests++;")
fcl.c_good_lines.append(
"\t\t} else if(WIFSIGNALED(returnCode) && WTERMSIG(returnCode) == SIGALRM) {"
)
fcl.c_good_lines.append("\t\t\tprintf(\"" + root + good +
"timed out\\n\");")
fcl.c_good_lines.append("\t\t\ttimedOutGoodTests++;")
fcl.c_good_lines.append("\t\t} else {")
fcl.c_good_lines.append("\t\t\tprintf(\"" + root + good +
" passed\\n\");")
fcl.c_good_lines.append("\t\t\tpassedGoodTests++;")
fcl.c_good_lines.append("\t\t}")
fcl.c_good_lines.append("\t} else {")
fcl.c_good_lines.append("\t\talarm(ALERT_TIMEOUT);")
fcl.c_good_lines.append("\t\t" + root + good +
"\n\t\texit(0);")
fcl.c_good_lines.append("\t}\n\ttotalGoodTests++;\n")
fcl.c_h_good_lines.append("\tvoid " + root + good + "\n")
else:
raise Exception(
"filename ends with something we don't handle!: " +
fullfilepath)
fcl.c_bad_lines.append(
"printf(\"Total Bad Tests: %d\\nPassed: %d\\nFailed: %d\\n\", totalBadTests, passedBadTests, failedBadTests);"
)
fcl.c_bad_lines.append(
"printf(\"Total Good Tests: %d\\nPassed: %d\\nFailed: %d\\n\", totalGoodTests, passedGoodTests, failedGoodTests);"
)
fcl.c_bad_lines.append(
"printf(\"Total Bad Time Outs: %d\\nTotal Good Time Outs: %d\\n\", timedOutBadTests, timedOutGoodTests);"
)
return fcl
update_main_cpp_and_testcases_h([cwe])
if __name__ == '__main__':
print("Working directory : " +
str(os.path.dirname(os.path.realpath(__file__))))
"""
Parse CWE command line argument
"""
options = parse_cmd()
"""
Iterate over CWE and run tests
"""
for cwe in options['cwe_list']:
py_common.print_with_timestamp("Starting test suite " + cwe + "...")
print(" [*] Generating test cases list")
test_cases = build_testcases_list(cwe, options['directory'])
print(" [*] Generating main_linux.cpp and testcases.h")
generate_dependencies(options['directory'], cwe)
# Adapte syntax to stay compliant with Clang frontend
print(" [*] Adapting C++ syntaxe")
adapte_namespace_syntax(test_cases.copy(), options['directory'])
adapte_main_linu_cpp_syntax(
cwe, options['directory'] + "./testcasesupport/main_linux.cpp")
print(" [*] Compiling test cases")
compiler = CWECompiler(test_cases, options['directory'],