def run_rebase_test(self, testfile):
testname = None
with zipfile.ZipFile(os.path.join(api_suite_path, testfile),
"r") as zf:
testname = zf.namelist()[0]
zf.extractall(path=api_suite_path)
pickle_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
testname + "_rebasing.pkl")
self.assertTrue(pickle_path, "Test pickle doesn't exist")
try:
# Python 2 does not have the encodings option
binary_oracle = pickle.load(open(pickle_path, "rb"),
encoding='charmap')
except TypeError:
binary_oracle = pickle.load(open(pickle_path, "rb"))
test_builder = testcommon.BinaryViewTestBuilder(testname,
imageBase=0xf00000)
self.assertTrue(test_builder.bv.start == 0xf00000)
for method in test_builder.methods():
test = getattr(test_builder, method)()
oracle = binary_oracle[method]
if test == oracle:
continue
result = getattr(test_builder, method).__doc__
result += ":\n"
report = self.report(oracle, test, result)
self.assertTrue(report[0],
report[1]) # Test does not agree with oracle
os.unlink(os.path.join(api_suite_path, testname))
def run_binary_test(self, testfile):
testname = None
with zipfile.ZipFile(testfile, "r") as zf:
testname = zf.namelist()[0]
zf.extractall()
self.assertTrue(os.path.exists(testname + ".pkl"),
"Test pickle doesn't exist")
try:
#Python 2 does not have the encodings option
binary_oracle = pickle.load(open(testname + ".pkl", "rUb"),
errors="ignore")
except TypeError:
binary_oracle = pickle.load(open(testname + ".pkl", "rU"))
test_builder = testcommon.BinaryViewTestBuilder(
testname, "suite/binaries/test_corpus")
for method in test_builder.methods():
test = getattr(test_builder, method)()
oracle = binary_oracle[method]
if test == oracle:
continue
result = getattr(test_builder, method).__doc__
result += ":\n"
d = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
skipped_lines = 0
for delta in d.compare(test, oracle):
if delta[0] == ' ':
skipped_lines += 1
continue
if skipped_lines > 0:
result += "<---" + str(skipped_lines) + ' same lines--->\n'
skipped_lines = 0
delta = delta.replace('\n', '')
result += delta + '\n'
self.assertTrue(False, result)
os.unlink(testname)
def generate(test_store, outdir, exclude_binaries):
if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)):
raise TestStoreError("Specified test store is not a directory")
unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store)
oracle = OracleTestFile(os.path.join(outdir, "oracle"))
# Generate the tests that don't involve binaries but do involve oracles
builder = testcommon.TestBuilder(test_store)
tests = builder.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating test data")
oracle.add_entry(builder, test_name)
unittest.add_test(test_name)
update_progress(len(tests), len(tests), "Generating test data", True)
# Generate the tests that just verify things work as expected
verify = testcommon.VerifyBuilder(test_store)
tests = verify.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating verify data")
unittest.add_verify(test_name)
update_progress(len(tests), len(tests), "Generating verify data", True)
# Now generate test that involve binaries
allfiles = sorted(testcommon.get_file_list(test_store))
for progress, testfile in enumerate(allfiles):
oraclefile = None
if testfile.endswith(".gitignore"):
continue
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".zip"):
# We have a zipped binary unzip it so we can rebaseline
with zipfile.ZipFile(testfile, "r") as zf:
zf.extractall(path = os.path.dirname(__file__))
if not os.path.exists(testfile[:-4]):
print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile))
continue
oraclefile = testfile[:-4]
else:
if os.path.exists(testfile + ".zip"):
# We've got a binary and zip for that binary just skip it
continue
# We have a binary that isn't zipped use it as a new test case
oraclefile = testfile
oraclefile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__))
# Now generate the oracle data
update_progress(progress, len(allfiles), oraclefile_rel)
unittest.add_binary_test(test_store, oraclefile_rel)
binary_start_time = time.time()
if exclude_binaries:
continue
test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel))
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
print("{0:.2f}".format(time.time() - binary_start_time))
# Generate oracle data for rebasing tests
name = oraclefile_rel[len(test_store):].replace(os.path.sep, "_").replace(".", "_")[1:]
if name in ["helloworld", "duff", "partial_register_dataflow", "raw"]:
test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel, imageBase=0xf00000)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel) + "_rebasing")
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
if not os.path.exists(oraclefile + ".zip"):
with zipfile.ZipFile(oraclefile + ".zip", "w") as zf:
zf.write(oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__)))
os.unlink(oraclefile)
update_progress(len(allfiles), len(allfiles), "Generating binary unit tests complete", True)
unittest.close()
oracle.close()
def generate(test_store, outdir, exclude_binaries, config_settings=None):
if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)):
raise TestStoreError("Specified test store is not a directory")
if not os.path.exists(outdir):
os.makedirs(outdir)
unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir,
test_store)
oracle = OracleTestFile(os.path.join(outdir, "oracle"))
# check all files to see if there is any newly added ones.
# If so, create a zip archive for it and delete the original file
allfiles = sorted(testcommon.get_file_list(test_store))
for progress, testfile in enumerate(allfiles):
oraclefile = None
zip_only = False
if testfile.endswith(".gitignore"):
continue
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".zip"):
continue
else:
if os.path.exists(testfile + ".zip"):
# We've got a zip file for it, skip
continue
# create the zip archive for the file
if not os.path.exists(testfile + ".zip"):
with zipfile.ZipFile(testfile + ".zip", "w") as zf:
zf.write(
testfile,
os.path.relpath(testfile,
start=os.path.dirname(__file__)))
os.unlink(testfile)
# Generate the tests that don't involve binaries but do involve oracles
builder = testcommon.TestBuilder(test_store)
tests = builder.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating test data")
oracle.add_entry(builder, test_name)
unittest.add_test(test_name)
update_progress(len(tests), len(tests), "Generating test data", True)
# Generate the tests that just verify things work as expected
verify = testcommon.VerifyBuilder(test_store)
tests = verify.methods()
for progress, test_name in enumerate(tests):
update_progress(progress, len(tests), "Generating verify data")
unittest.add_verify(test_name)
update_progress(len(tests), len(tests), "Generating verify data", True)
# Now generate test that involve binaries
allfiles = sorted(testcommon.get_file_list(test_store))
total_progress = len(allfiles)
for progress, testfile in enumerate(allfiles):
oraclefile = None
zip_only = False
if testfile.endswith(".gitignore"):
continue
if testfile.endswith(".pkl"):
continue
elif testfile.endswith(".DS_Store"):
continue
elif testfile.endswith(".bndb"):
# For databases, we do not wish to create oracle data for them
# However, we do wish them to be zipped
zip_only = True
oraclefile = testfile
elif testfile.endswith(".bndb.zip"):
continue
elif testfile.endswith(".zip"):
# We have a zipped binary unzip it so we can rebaseline
with zipfile.ZipFile(testfile, "r") as zf:
zf.extractall(path=os.path.dirname(__file__))
if not os.path.exists(testfile[:-4]):
print("Error extracting testfile %s from zip: %s" %
(testfile[:-4], testfile))
continue
oraclefile = testfile[:-4]
else:
if os.path.exists(testfile + ".zip"):
# We've got a binary and zip for that binary just skip it
continue
# We have a binary that isn't zipped use it as a new test case
oraclefile = testfile
# create the zip archive for the file
if not os.path.exists(oraclefile + ".zip"):
with zipfile.ZipFile(oraclefile + ".zip", "w") as zf:
zf.write(
oraclefile,
os.path.relpath(oraclefile,
start=os.path.dirname(__file__)))
if zip_only:
os.unlink(oraclefile)
continue
testfile_basename = os.path.basename(oraclefile)
testfile_rel = os.path.relpath(oraclefile,
start=os.path.dirname(__file__))
oraclefile_basepath = testfile_rel[:-len(testfile_basename)]
oraclefile_rel = os.path.join(oraclefile_basepath, testfile_basename)
# Create directory for pickle oracle results
if not os.path.exists(os.path.join(outdir, oraclefile_basepath)):
os.makedirs(os.path.join(outdir, oraclefile_basepath))
# Now generate the oracle data
update_progress(progress, len(allfiles), oraclefile_rel)
unittest.add_binary_test(test_store,
testfile_rel,
config_settings=config_settings)
binary_start_time = time.time()
if exclude_binaries:
continue
test_data = testcommon.BinaryViewTestBuilder(testfile_rel,
config_settings)
binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel))
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
print("{0:.2f}".format(time.time() - binary_start_time))
# Generate oracle data for rebasing tests
if testfile_basename in [
"helloworld", "duff", "partial_register_dataflow", "raw"
]:
oracle_suffix = "_rebasing"
rebasing_options = {
**config_settings,
**{
'loader.imageBase': 0xf00000
}
}
unittest.add_binary_test(test_store, testfile_rel, oracle_suffix,
rebasing_options)
test_data = testcommon.BinaryViewTestBuilder(
testfile_rel, rebasing_options)
binary_oracle = OracleTestFile(
os.path.join(outdir, oraclefile_rel) + oracle_suffix)
for method in test_data.methods():
binary_oracle.add_entry(test_data, method)
binary_oracle.close()
os.unlink(oraclefile)
update_progress(total_progress, total_progress,
"Generating binary unit tests complete", True)
unittest.close()
oracle.close()
