def main():
usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]"
parser = OptionParser(usage=usage)
parser.add_option("-q", "--quiet",
dest="quiet", action="store_true",
default=False, help="Don't print anything")
parser.add_option("-x", "--exclude",
dest="exclude_binary", action="store_true",
default=False, help="Exclude regeneration of binaries")
parser.add_option("-o", "--outputdir", default="suite",
dest="outputdir", action="store", type="string",
help="output directory where the unit.py and oracle.py files will be stored")
parser.add_option("-i", "--inputdir", default=os.path.join("suite", "binaries", "test_corpus"),
dest="test_store", action="store", type="string",
help="input directory containing the binaries you which to generate unit tests from")
options, args = parser.parse_args()
if not os.path.exists(os.path.join(os.getcwd(), 'suite')):
print("Error: Please run this script from the binaryninja-api root directory")
sys.exit(1)
myprint("[+] INFO: Using test store: %s" % options.test_store)
if len(testcommon.get_file_list(options.test_store)) == 0:
myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store))
sys.exit(1)
myprint("[+] INFO: Generating test store")
try:
generate(options.test_store, options.outputdir, options.exclude_binary)
except TestStoreError as te:
myprint("[-] ERROR: Failed to generate test store: %s" % te.message)
sys.exit(1)
myprint("[+] SUCCESS: Generating test store")
sys.exit(0)
def main():
usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]"
parser = OptionParser(usage=usage)
default_output = os.path.relpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "suite"))
parser.add_option("-q", "--quiet",
dest="quiet", action="store_true",
default=False, help="Don't print anything")
parser.add_option("-x", "--exclude",
dest="exclude_binary", action="store_true",
default=False, help="Exclude regeneration of binaries")
parser.add_option("-o", "--outputdir", default=default_output,
dest="outputdir", action="store", type="string",
help="output directory where the unit.py and oracle.py files will be stored (relative to cwd)")
parser.add_option("-i", "--inputdir", default=os.path.join("binaries", "test_corpus"),
dest="test_store", action="store", type="string",
help="input directory containing the binaries you which to generate unit tests from (relative to this file)")
options, args = parser.parse_args()
print("OUTPUT: %s" % options.outputdir)
myprint("[+] INFO: Using test store: %s" % options.test_store)
if len(testcommon.get_file_list(options.test_store)) == 0:
myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store))
sys.exit(1)
myprint("[+] INFO: Generating test store")
try:
generate(options.test_store, options.outputdir, options.exclude_binary)
except TestStoreError as te:
myprint("[-] ERROR: Failed to generate test store: %s" % te.message)
sys.exit(1)
myprint("[+] SUCCESS: Generating test store")
sys.exit(0)
def main():
usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]"
parser = OptionParser(usage=usage)
parser.add_option("-q", "--quiet",
dest="quiet", action="store_true",
default=False, help="Don't print anything")
parser.add_option("-x", "--exclude",
dest="exclude_binary", action="store_true",
default=False, help="Exclude regeneration of binaries")
parser.add_option("-o", "--outputdir", default="suite",
dest="outputdir", action="store", type="string",
help="output directory where the unit.py and oracle.py files will be stored (relative to cwd)")
parser.add_option("-i", "--inputdir", default=os.path.join("binaries", "test_corpus"),
dest="test_store", action="store", type="string",
help="input directory containing the binaries you which to generate unit tests from (relative to this file)")
options, args = parser.parse_args()
myprint("[+] INFO: Using test store: %s" % options.test_store)
if len(testcommon.get_file_list(options.test_store)) == 0:
myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store))
sys.exit(1)
myprint("[+] INFO: Generating test store")
try:
generate(options.test_store, options.outputdir, options.exclude_binary)
except TestStoreError as te:
myprint("[-] ERROR: Failed to generate test store: %s" % te.message)
sys.exit(1)
myprint("[+] SUCCESS: Generating test store")
sys.exit(0)
def generate(test_store, outdir, exclude_binaries):
if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)):
raise TestStoreError("Specified test store is not a directory")
unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store)
oracle = OracleTestFile(os.path.join(outdir, "oracle"))
# Generate the tests that don't involve binaries but do involve oracles
builder = testcommon.TestBuilder(test_store)
tests = builder.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating test data")
oracle.add_entry(builder, test_name)
unittest.add_test(test_name)
update_progress(len(tests), len(tests), "Generating test data", True)
# Generate the tests that just verify things work as expected
verify = testcommon.VerifyBuilder(test_store)
tests = verify.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating verify data")
unittest.add_verify(test_name)
update_progress(len(tests), len(tests), "Generating verify data", True)
# Now generate test that involve binaries
allfiles = sorted(testcommon.get_file_list(test_store))
for progress, testfile in enumerate(allfiles):
oraclefile = None
if testfile.endswith(".gitignore"):
continue
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".zip"):
# We have a zipped binary unzip it so we can rebaseline
with zipfile.ZipFile(testfile, "r") as zf:
zf.extractall(path = os.path.dirname(__file__))
if not os.path.exists(testfile[:-4]):
print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile))
continue
oraclefile = testfile[:-4]
else:
if os.path.exists(testfile + ".zip"):
# We've got a binary and zip for that binary just skip it
continue
# We have a binary that isn't zipped use it as a new test case
oraclefile = testfile
oraclefile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__))
# Now generate the oracle data
update_progress(progress, len(allfiles), oraclefile_rel)
unittest.add_binary_test(test_store, oraclefile_rel)
binary_start_time = time.time()
if exclude_binaries:
continue
test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel))
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
print("{0:.2f}".format(time.time() - binary_start_time))
# Generate oracle data for rebasing tests
name = oraclefile_rel[len(test_store):].replace(os.path.sep, "_").replace(".", "_")[1:]
if name in ["helloworld", "duff", "partial_register_dataflow", "raw"]:
test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel, imageBase=0xf00000)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel) + "_rebasing")
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
if not os.path.exists(oraclefile + ".zip"):
with zipfile.ZipFile(oraclefile + ".zip", "w") as zf:
zf.write(oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__)))
os.unlink(oraclefile)
update_progress(len(allfiles), len(allfiles), "Generating binary unit tests complete", True)
unittest.close()
oracle.close()
def main():
usage = "usage: %prog [-q] [-x] [-o <dir>] [-i <dir>]"
parser = OptionParser(usage=usage)
default_output = os.path.relpath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "suite",
"generated"))
parser.add_option("-q",
"--quiet",
dest="quiet",
action="store_true",
default=False,
help="Don't print anything")
parser.add_option("-x",
"--exclude",
dest="exclude_binary",
action="store_true",
default=False,
help="Exclude regeneration of binaries")
parser.add_option(
"-o",
"--outputdir",
default=default_output,
dest="outputdir",
action="store",
type="string",
help=
"output directory where the unit.py and oracle.py files will be stored (relative to cwd)"
)
parser.add_option(
"-i",
"--inputdir",
default=os.path.join("binaries", "test_corpus"),
dest="test_store",
action="store",
type="string",
help=
"input directory containing the binaries you which to generate unit tests from (relative to this file)"
)
parser.add_option(
"-a",
"--analysismodes",
dest="analysis_modes",
action="store_true",
default=False,
help="Generate additional oracle files to support analysis mode testing"
)
options, args = parser.parse_args()
test_store_location = os.path.relpath(
os.path.join(testcommon.BinaryViewTestBuilder.get_root_directory(),
options.test_store))
myprint(f"[+] INFO: Input Test Corpus: {test_store_location}")
if len(testcommon.get_file_list(options.test_store)) == 0:
myprint(
f"ERROR: Test Corpus is empty: {testcommon.get_file_list(options.test_store)}"
)
sys.exit(1)
configurations = {}
configurations['default'] = {}
if options.analysis_modes:
configurations['mode_controlflow'] = {'analysis.mode': 'controlFlow'}
configurations['mode_basic'] = {'analysis.mode': 'basic'}
configurations['mode_intermediate'] = {'analysis.mode': 'intermediate'}
myprint("[+] INFO: Generating Automated Unit Tests and Oracle Results")
for (name, config_settings) in configurations.items():
oracle_target = os.path.join(options.outputdir, name)
myprint(f"[+] INFO: Oracle Target Directory: {oracle_target}")
try:
generate(options.test_store, oracle_target, options.exclude_binary,
config_settings)
myprint(
f"[+] SUCCESS: Generated Results for the '{name}' Configuration"
)
except TestStoreError as te:
myprint(
f"[-] ERROR: Failed to Generate Results for the '{name}' Configuration: {te.message}"
)
sys.exit(1)
sys.exit(0)
def generate(test_store, outdir, exclude_binaries, config_settings=None):
if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)):
raise TestStoreError("Specified test store is not a directory")
if not os.path.exists(outdir):
os.makedirs(outdir)
unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir,
test_store)
oracle = OracleTestFile(os.path.join(outdir, "oracle"))
# check all files to see if there is any newly added ones.
# If so, create a zip archive for it and delete the original file
allfiles = sorted(testcommon.get_file_list(test_store))
for progress, testfile in enumerate(allfiles):
oraclefile = None
zip_only = False
if testfile.endswith(".gitignore"):
continue
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".zip"):
continue
else:
if os.path.exists(testfile + ".zip"):
# We've got a zip file for it, skip
continue
# create the zip archive for the file
if not os.path.exists(testfile + ".zip"):
with zipfile.ZipFile(testfile + ".zip", "w") as zf:
zf.write(
testfile,
os.path.relpath(testfile,
start=os.path.dirname(__file__)))
os.unlink(testfile)
# Generate the tests that don't involve binaries but do involve oracles
builder = testcommon.TestBuilder(test_store)
tests = builder.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating test data")
oracle.add_entry(builder, test_name)
unittest.add_test(test_name)
update_progress(len(tests), len(tests), "Generating test data", True)
# Generate the tests that just verify things work as expected
verify = testcommon.VerifyBuilder(test_store)
tests = verify.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating verify data")
unittest.add_verify(test_name)
update_progress(len(tests), len(tests), "Generating verify data", True)
# Now generate test that involve binaries
allfiles = sorted(testcommon.get_file_list(test_store))
total_progress = len(allfiles)
for progress, testfile in enumerate(allfiles):
oraclefile = None
zip_only = False
if testfile.endswith(".gitignore"):
continue
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".bndb"):
# For databases, we do not wish to create oracle data for them
# However, we do wish them to be zipped
zip_only = True
oraclefile = testfile
elif testfile.endswith(".bndb.zip"):
continue
elif testfile.endswith(".zip"):
# We have a zipped binary unzip it so we can rebaseline
with zipfile.ZipFile(testfile, "r") as zf:
zf.extractall(path=os.path.dirname(__file__))
if not os.path.exists(testfile[:-4]):
print("Error extracting testfile %s from zip: %s" %
(testfile[:-4], testfile))
continue
oraclefile = testfile[:-4]
else:
if os.path.exists(testfile + ".zip"):
# We've got a binary and zip for that binary just skip it
continue
# We have a binary that isn't zipped use it as a new test case
oraclefile = testfile
# create the zip archive for the file
if not os.path.exists(oraclefile + ".zip"):
with zipfile.ZipFile(oraclefile + ".zip", "w") as zf:
zf.write(
oraclefile,
os.path.relpath(oraclefile,
start=os.path.dirname(__file__)))
if zip_only:
os.unlink(oraclefile)
continue
testfile_basename = os.path.basename(oraclefile)
testfile_rel = os.path.relpath(oraclefile,
start=os.path.dirname(__file__))
oraclefile_basepath = testfile_rel[:-len(testfile_basename)]
oraclefile_rel = os.path.join(oraclefile_basepath, testfile_basename)
# Create directory for pickle oracle results
if not os.path.exists(os.path.join(outdir, oraclefile_basepath)):
os.makedirs(os.path.join(outdir, oraclefile_basepath))
# Now generate the oracle data
update_progress(progress, len(allfiles), oraclefile_rel)
unittest.add_binary_test(test_store,
testfile_rel,
config_settings=config_settings)
binary_start_time = time.time()
if exclude_binaries:
continue
test_data = testcommon.BinaryViewTestBuilder(testfile_rel,
config_settings)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel))
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
print("{0:.2f}".format(time.time() - binary_start_time))
# Generate oracle data for rebasing tests
if testfile_basename in [
"helloworld", "duff", "partial_register_dataflow", "raw"
]:
oracle_suffix = "_rebasing"
rebasing_options = {
**config_settings,
**{
'loader.imageBase': 0xf00000
}
}
unittest.add_binary_test(test_store, testfile_rel, oracle_suffix,
rebasing_options)
test_data = testcommon.BinaryViewTestBuilder(
testfile_rel, rebasing_options)
binary_oracle = OracleTestFile(
os.path.join(outdir, oraclefile_rel) + oracle_suffix)
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
os.unlink(oraclefile)
update_progress(total_progress, total_progress,
"Generating binary unit tests complete", True)
unittest.close()
oracle.close()
def generate(test_store, outdir, exclude_binaries):
if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)):
raise TestStoreError("Specified test store is not a directory")
unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store)
oracle = OracleTestFile(os.path.join(outdir, "oracle"))
# Generate the tests that don't involve binaries but do involve oracles
builder = testcommon.TestBuilder(test_store)
tests = builder.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating test data")
oracle.add_entry(builder, test_name)
unittest.add_test(test_name)
update_progress(len(tests), len(tests), "Generating test data", True)
# Generate the tests that just verify things work as expected
verify = testcommon.VerifyBuilder(test_store)
tests = verify.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating verify data")
unittest.add_verify(test_name)
update_progress(len(tests), len(tests), "Generating verify data", True)
# Now generate test that involve binaries
allfiles = sorted(testcommon.get_file_list(test_store))
for progress, testfile in enumerate(allfiles):
oraclefile = None
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".zip"):
# We have a zipped binary unzip it so we can rebaseline
with zipfile.ZipFile(testfile, "r") as zf:
zf.extractall(path = os.path.dirname(__file__))
if not os.path.exists(testfile[:-4]):
print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile))
continue
oraclefile = testfile[:-4]
else:
if os.path.exists(testfile + ".zip"):
# We've got a binary and zip for that binary just skip it
continue
# We have a binary that isn't zipped use it as a new test case
oraclefile = testfile
oraclefile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__))
# Now generate the oracle data
update_progress(progress, len(allfiles), oraclefile_rel)
unittest.add_binary_test(test_store, oraclefile_rel)
binary_start_time = time.time()
if exclude_binaries:
continue
test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel))
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
print("{0:.2f}".format(time.time() - binary_start_time))
if not os.path.exists(oraclefile + ".zip"):
with zipfile.ZipFile(oraclefile + ".zip", "w") as zf:
zf.write(oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__)))
os.unlink(oraclefile)
update_progress(len(allfiles), len(allfiles), "Generating binary unit tests complete", True)
unittest.close()
oracle.close()