def main(): usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]" parser = OptionParser(usage=usage) parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything") parser.add_option("-x", "--exclude", dest="exclude_binary", action="store_true", default=False, help="Exclude regeneration of binaries") parser.add_option("-o", "--outputdir", default="suite", dest="outputdir", action="store", type="string", help="output directory where the unit.py and oracle.py files will be stored") parser.add_option("-i", "--inputdir", default=os.path.join("suite", "binaries", "test_corpus"), dest="test_store", action="store", type="string", help="input directory containing the binaries you which to generate unit tests from") options, args = parser.parse_args() if not os.path.exists(os.path.join(os.getcwd(), 'suite')): print("Error: Please run this script from the binaryninja-api root directory") sys.exit(1) myprint("[+] INFO: Using test store: %s" % options.test_store) if len(testcommon.get_file_list(options.test_store)) == 0: myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store)) sys.exit(1) myprint("[+] INFO: Generating test store") try: generate(options.test_store, options.outputdir, options.exclude_binary) except TestStoreError as te: myprint("[-] ERROR: Failed to generate test store: %s" % te.message) sys.exit(1) myprint("[+] SUCCESS: Generating test store") sys.exit(0)
def main(): usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]" parser = OptionParser(usage=usage) default_output = os.path.relpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "suite")) parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything") parser.add_option("-x", "--exclude", dest="exclude_binary", action="store_true", default=False, help="Exclude regeneration of binaries") parser.add_option("-o", "--outputdir", default=default_output, dest="outputdir", action="store", type="string", help="output directory where the unit.py and oracle.py files will be stored (relative to cwd)") parser.add_option("-i", "--inputdir", default=os.path.join("binaries", "test_corpus"), dest="test_store", action="store", type="string", help="input directory containing the binaries you which to generate unit tests from (relative to this file)") options, args = parser.parse_args() print("OUTPUT: %s" % options.outputdir) myprint("[+] INFO: Using test store: %s" % options.test_store) if len(testcommon.get_file_list(options.test_store)) == 0: myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store)) sys.exit(1) myprint("[+] INFO: Generating test store") try: generate(options.test_store, options.outputdir, options.exclude_binary) except TestStoreError as te: myprint("[-] ERROR: Failed to generate test store: %s" % te.message) sys.exit(1) myprint("[+] SUCCESS: Generating test store") sys.exit(0)
def main(): usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]" parser = OptionParser(usage=usage) parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything") parser.add_option("-x", "--exclude", dest="exclude_binary", action="store_true", default=False, help="Exclude regeneration of binaries") parser.add_option("-o", "--outputdir", default="suite", dest="outputdir", action="store", type="string", help="output directory where the unit.py and oracle.py files will be stored (relative to cwd)") parser.add_option("-i", "--inputdir", default=os.path.join("binaries", "test_corpus"), dest="test_store", action="store", type="string", help="input directory containing the binaries you which to generate unit tests from (relative to this file)") options, args = parser.parse_args() myprint("[+] INFO: Using test store: %s" % options.test_store) if len(testcommon.get_file_list(options.test_store)) == 0: myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store)) sys.exit(1) myprint("[+] INFO: Generating test store") try: generate(options.test_store, options.outputdir, options.exclude_binary) except TestStoreError as te: myprint("[-] ERROR: Failed to generate test store: %s" % te.message) sys.exit(1) myprint("[+] SUCCESS: Generating test store") sys.exit(0)
def generate(test_store, outdir, exclude_binaries): if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)): raise TestStoreError("Specified test store is not a directory") unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store) oracle = OracleTestFile(os.path.join(outdir, "oracle")) # Generate the tests that don't involve binaries but do involve oracles builder = testcommon.TestBuilder(test_store) tests = builder.methods() for progress, test_name in enumerate(tests): update_progress(progress, len(tests), "Generating test data") oracle.add_entry(builder, test_name) unittest.add_test(test_name) update_progress(len(tests), len(tests), "Generating test data", True) # Generate the tests that just verify things work as expected verify = testcommon.VerifyBuilder(test_store) tests = verify.methods() for progress, test_name in enumerate(tests): update_progress(progress, len(tests), "Generating verify data") unittest.add_verify(test_name) update_progress(len(tests), len(tests), "Generating verify data", True) # Now generate test that involve binaries allfiles = sorted(testcommon.get_file_list(test_store)) for progress, testfile in enumerate(allfiles): oraclefile = None if testfile.endswith(".gitignore"): continue if testfile.endswith(".pkl"): continue elif testfile.endswith(".DS_Store"): continue elif testfile.endswith(".zip"): # We have a zipped binary unzip it so we can rebaseline with zipfile.ZipFile(testfile, "r") as zf: zf.extractall(path = os.path.dirname(__file__)) if not os.path.exists(testfile[:-4]): print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile)) continue oraclefile = testfile[:-4] else: if os.path.exists(testfile + ".zip"): # We've got a binary and zip for that binary just skip it continue # We have a binary that isn't zipped use it as a new test case oraclefile = testfile oraclefile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__)) # Now generate the oracle data update_progress(progress, len(allfiles), oraclefile_rel) unittest.add_binary_test(test_store, oraclefile_rel) binary_start_time = time.time() if exclude_binaries: continue test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel) binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel)) for method in test_data.methods(): binary_oracle.add_entry(test_data, method) binary_oracle.close() print("{0:.2f}".format(time.time() - binary_start_time)) # Generate oracle data for rebasing tests name = oraclefile_rel[len(test_store):].replace(os.path.sep, "_").replace(".", "_")[1:] if name in ["helloworld", "duff", "partial_register_dataflow", "raw"]: test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel, imageBase=0xf00000) binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel) + "_rebasing") for method in test_data.methods(): binary_oracle.add_entry(test_data, method) binary_oracle.close() if not os.path.exists(oraclefile + ".zip"): with zipfile.ZipFile(oraclefile + ".zip", "w") as zf: zf.write(oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__))) os.unlink(oraclefile) update_progress(len(allfiles), len(allfiles), "Generating binary unit tests complete", True) unittest.close() oracle.close()
def main(): usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]" parser = OptionParser(usage=usage) default_output = os.path.relpath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "suite", "generated")) parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything") parser.add_option("-x", "--exclude", dest="exclude_binary", action="store_true", default=False, help="Exclude regeneration of binaries") parser.add_option( "-o", "--outputdir", default=default_output, dest="outputdir", action="store", type="string", help= "output directory where the unit.py and oracle.py files will be stored (relative to cwd)" ) parser.add_option( "-i", "--inputdir", default=os.path.join("binaries", "test_corpus"), dest="test_store", action="store", type="string", help= "input directory containing the binaries you which to generate unit tests from (relative to this file)" ) parser.add_option( "-a", "--analysismodes", dest="analysis_modes", action="store_true", default=False, help="Generate additional oracle files to support analysis mode testing" ) options, args = parser.parse_args() test_store_location = os.path.relpath( os.path.join(testcommon.BinaryViewTestBuilder.get_root_directory(), options.test_store)) myprint(f"[+] INFO: Input Test Corpus: {test_store_location}") if len(testcommon.get_file_list(options.test_store)) == 0: myprint( f"ERROR: Test Corpus is empty: {testcommon.get_file_list(options.test_store)}" ) sys.exit(1) configurations = {} configurations['default'] = {} if options.analysis_modes: configurations['mode_controlflow'] = {'analysis.mode': 'controlFlow'} configurations['mode_basic'] = {'analysis.mode': 'basic'} configurations['mode_intermediate'] = {'analysis.mode': 'intermediate'} myprint("[+] INFO: Generating Automated Unit Tests and Oracle Results") for (name, config_settings) in configurations.items(): oracle_target = os.path.join(options.outputdir, name) myprint(f"[+] INFO: Oracle Target Directory: {oracle_target}") try: generate(options.test_store, oracle_target, options.exclude_binary, config_settings) myprint( f"[+] SUCCESS: Generated Results for the '{name}' Configuration" ) except TestStoreError as te: myprint( f"[-] ERROR: Failed to Generate Results for the '{name}' Configuration: {te.message}" ) sys.exit(1) sys.exit(0)
def generate(test_store, outdir, exclude_binaries, config_settings=None): if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)): raise TestStoreError("Specified test store is not a directory") if not os.path.exists(outdir): os.makedirs(outdir) unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store) oracle = OracleTestFile(os.path.join(outdir, "oracle")) # check all files to see if there is any newly added ones. # If so, create a zip archive for it and delete the original file allfiles = sorted(testcommon.get_file_list(test_store)) for progress, testfile in enumerate(allfiles): oraclefile = None zip_only = False if testfile.endswith(".gitignore"): continue if testfile.endswith(".pkl"): continue elif testfile.endswith(".DS_Store"): continue elif testfile.endswith(".zip"): continue else: if os.path.exists(testfile + ".zip"): # We've got a zip file for it, skip continue # create the zip archive for the file if not os.path.exists(testfile + ".zip"): with zipfile.ZipFile(testfile + ".zip", "w") as zf: zf.write( testfile, os.path.relpath(testfile, start=os.path.dirname(__file__))) os.unlink(testfile) # Generate the tests that don't involve binaries but do involve oracles builder = testcommon.TestBuilder(test_store) tests = builder.methods() for progress, test_name in enumerate(tests): update_progress(progress, len(tests), "Generating test data") oracle.add_entry(builder, test_name) unittest.add_test(test_name) update_progress(len(tests), len(tests), "Generating test data", True) # Generate the tests that just verify things work as expected verify = testcommon.VerifyBuilder(test_store) tests = verify.methods() for progress, test_name in enumerate(tests): update_progress(progress, len(tests), "Generating verify data") unittest.add_verify(test_name) update_progress(len(tests), len(tests), "Generating verify data", True) # Now generate test that involve binaries allfiles = sorted(testcommon.get_file_list(test_store)) total_progress = len(allfiles) for progress, testfile in enumerate(allfiles): oraclefile = None zip_only = False if testfile.endswith(".gitignore"): continue if testfile.endswith(".pkl"): continue elif testfile.endswith(".DS_Store"): continue elif testfile.endswith(".bndb"): # For databases, we do not wish to create oracle data for them # However, we do wish them to be zipped zip_only = True oraclefile = testfile elif testfile.endswith(".bndb.zip"): continue elif testfile.endswith(".zip"): # We have a zipped binary unzip it so we can rebaseline with zipfile.ZipFile(testfile, "r") as zf: zf.extractall(path=os.path.dirname(__file__)) if not os.path.exists(testfile[:-4]): print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile)) continue oraclefile = testfile[:-4] else: if os.path.exists(testfile + ".zip"): # We've got a binary and zip for that binary just skip it continue # We have a binary that isn't zipped use it as a new test case oraclefile = testfile # create the zip archive for the file if not os.path.exists(oraclefile + ".zip"): with zipfile.ZipFile(oraclefile + ".zip", "w") as zf: zf.write( oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__))) if zip_only: os.unlink(oraclefile) continue testfile_basename = os.path.basename(oraclefile) testfile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__)) oraclefile_basepath = testfile_rel[:-len(testfile_basename)] oraclefile_rel = os.path.join(oraclefile_basepath, testfile_basename) # Create directory for pickle oracle results if not os.path.exists(os.path.join(outdir, oraclefile_basepath)): os.makedirs(os.path.join(outdir, oraclefile_basepath)) # Now generate the oracle data update_progress(progress, len(allfiles), oraclefile_rel) unittest.add_binary_test(test_store, testfile_rel, config_settings=config_settings) binary_start_time = time.time() if exclude_binaries: continue test_data = testcommon.BinaryViewTestBuilder(testfile_rel, config_settings) binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel)) for method in test_data.methods(): binary_oracle.add_entry(test_data, method) binary_oracle.close() print("{0:.2f}".format(time.time() - binary_start_time)) # Generate oracle data for rebasing tests if testfile_basename in [ "helloworld", "duff", "partial_register_dataflow", "raw" ]: oracle_suffix = "_rebasing" rebasing_options = { **config_settings, **{ 'loader.imageBase': 0xf00000 } } unittest.add_binary_test(test_store, testfile_rel, oracle_suffix, rebasing_options) test_data = testcommon.BinaryViewTestBuilder( testfile_rel, rebasing_options) binary_oracle = OracleTestFile( os.path.join(outdir, oraclefile_rel) + oracle_suffix) for method in test_data.methods(): binary_oracle.add_entry(test_data, method) binary_oracle.close() os.unlink(oraclefile) update_progress(total_progress, total_progress, "Generating binary unit tests complete", True) unittest.close() oracle.close()
def generate(test_store, outdir, exclude_binaries): if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)): raise TestStoreError("Specified test store is not a directory") unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store) oracle = OracleTestFile(os.path.join(outdir, "oracle")) # Generate the tests that don't involve binaries but do involve oracles builder = testcommon.TestBuilder(test_store) tests = builder.methods() for progress, test_name in enumerate(tests): update_progress(progress, len(tests), "Generating test data") oracle.add_entry(builder, test_name) unittest.add_test(test_name) update_progress(len(tests), len(tests), "Generating test data", True) # Generate the tests that just verify things work as expected verify = testcommon.VerifyBuilder(test_store) tests = verify.methods() for progress, test_name in enumerate(tests): update_progress(progress, len(tests), "Generating verify data") unittest.add_verify(test_name) update_progress(len(tests), len(tests), "Generating verify data", True) # Now generate test that involve binaries allfiles = sorted(testcommon.get_file_list(test_store)) for progress, testfile in enumerate(allfiles): oraclefile = None if testfile.endswith(".pkl"): continue elif testfile.endswith(".DS_Store"): continue elif testfile.endswith(".zip"): # We have a zipped binary unzip it so we can rebaseline with zipfile.ZipFile(testfile, "r") as zf: zf.extractall(path = os.path.dirname(__file__)) if not os.path.exists(testfile[:-4]): print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile)) continue oraclefile = testfile[:-4] else: if os.path.exists(testfile + ".zip"): # We've got a binary and zip for that binary just skip it continue # We have a binary that isn't zipped use it as a new test case oraclefile = testfile oraclefile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__)) # Now generate the oracle data update_progress(progress, len(allfiles), oraclefile_rel) unittest.add_binary_test(test_store, oraclefile_rel) binary_start_time = time.time() if exclude_binaries: continue test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel) binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel)) for method in test_data.methods(): binary_oracle.add_entry(test_data, method) binary_oracle.close() print("{0:.2f}".format(time.time() - binary_start_time)) if not os.path.exists(oraclefile + ".zip"): with zipfile.ZipFile(oraclefile + ".zip", "w") as zf: zf.write(oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__))) os.unlink(oraclefile) update_progress(len(allfiles), len(allfiles), "Generating binary unit tests complete", True) unittest.close() oracle.close()