def test_diff_test(self):
"""Tests for the 'diff_test' interestingness test"""
l = lithium.Lithium() # noqa: E741
with open("temp.js", "w"):
pass
# test that the parameters "-a" and "-b" of diff_test work
result = l.main([
"diff_test", "--timeout", "99", "-a", "flags_one", "-b",
"flags_two_a flags_two_b"
] + self.ls_cmd + ["temp.js"])
self.assertEqual(result, 0)
result = l.main([
"diff_test", "--a-args", "flags_one_a flags_one_b", "--b-args",
"flags_two"
] + self.ls_cmd + ["temp.js"])
self.assertEqual(result, 0)
def test_interestingness_outputs_multiline(capsys, pattern, expected):
"""Tests for the 'outputs' interestingness test with multiline pattern"""
lith = lithium.Lithium()
with open("temp.js", "wb") as tmp_f:
tmp_f.write(b"line A\nline B\nline C\nline D\nline E\n")
capsys.readouterr() # clear captured output buffers
result = lith.main([
"outputs",
pattern,
] + CAT_CMD + ["temp.js"])
assert result == 0, "%r not found in %r" % (pattern,
open("temp.js").read())
# assert lith.test_count == 1
captured = capsys.readouterr()
assert "[Found string in: %r]" % (expected, ) in captured.out
def test_crashes_2(examples_path: Path) -> None:
"""crash test for the 'crashes' interestingness test"""
lith = lithium.Lithium()
# if a compiler is available, compile a simple crashing test program
src = examples_path / "crash.c"
exe = Path.cwd().resolve() / ("crash.exe" if platform.system() == "Windows"
else "crash")
try:
_compile(src, exe)
except RuntimeError as exc:
LOG.warning(exc)
pytest.skip("compile 'crash.c' failed")
result = lith.main(
["--strategy", "check-only", "crashes",
str(exe), "temp.js"])
assert result == 0
assert lith.test_count == 1
def test_diff_test_0() -> None:
"""test for the 'diff_test' interestingness test"""
lith = lithium.Lithium()
# test that the parameters "-a" and "-b" of diff_test work
result = lith.main([
"--strategy",
"check-only",
"diff_test",
"--timeout",
"99",
"-a",
"flags_one",
"-b",
"flags_two_a flags_two_b",
] + LS_CMD + ["temp.js"])
assert result == 0
assert lith.test_count == 1
def test_minimize(self):
class Interesting(DummyInteresting):
def interesting(sub, conditionArgs, tempPrefix): # pylint: disable=no-self-argument
# pylint: disable=missing-return-doc,missing-return-type-doc
with open("a.txt", "rb") as f:
return b"o\n" in f.read()
l = lithium.Lithium()
l.conditionScript = Interesting()
l.strategy = lithium.Minimize()
for testcaseType in (lithium.TestcaseChar, lithium.TestcaseLine, lithium.TestcaseSymbol):
log.info("Trying with testcase type %s:", testcaseType.__name__)
with open("a.txt", "wb") as f:
f.write(b"x\n\nx\nx\no\nx\nx\nx\n")
l.testcase = testcaseType()
l.testcase.readTestcase("a.txt")
self.assertEqual(l.run(), 0)
with open("a.txt", "rb") as f:
self.assertEqual(f.read(), b"o\n")
def test_crashes_1() -> None:
"""timeout test for the 'crashes' interestingness test"""
lith = lithium.Lithium()
# check that --timeout works
start_time = time.time()
result = lith.main([
"--strategy",
"check-only",
"--testcase",
"temp.js",
"crashes",
"--timeout",
"1",
] + SLEEP_CMD + ["3"])
elapsed = time.time() - start_time
assert result == 1
assert elapsed >= 1
assert lith.test_count == 1
def test_replace_properties(self):
valid_reductions = (
# original: this.list, prototype.push, prototype.last
b"function Foo() {\n this.list = [];\n}\nFoo.prototype.push = function(a) {\n this.list.push(a);\n}\nFoo.prototype.last = function() {\n return this.list.pop();\n}\n",
# this.list, prototype.push, last
b"function Foo() {\n this.list = [];\n}\nFoo.prototype.push = function(a) {\n this.list.push(a);\n}\nlast = function() {\n return this.list.pop();\n}\n",
# this.list, push, prototype.last
b"function Foo() {\n this.list = [];\n}\npush = function(a) {\n this.list.push(a);\n}\nFoo.prototype.last = function() {\n return this.list.pop();\n}\n",
# this.list, push, last
b"function Foo() {\n this.list = [];\n}\npush = function(a) {\n this.list.push(a);\n}\nlast = function() {\n return this.list.pop();\n}\n",
# list, prototype.push, prototype.last
b"function Foo() {\n list = [];\n}\nFoo.prototype.push = function(a) {\n list.push(a);\n}\nFoo.prototype.last = function() {\n return list.pop();\n}\n",
# list, prototype.push, last
b"function Foo() {\n list = [];\n}\nFoo.prototype.push = function(a) {\n list.push(a);\n}\nlast = function() {\n return list.pop();\n}\n",
# list, push, prototype.last
b"function Foo() {\n list = [];\n}\npush = function(a) {\n list.push(a);\n}\nFoo.prototype.last = function() {\n return list.pop();\n}\n",
# reduced: list, push, last
b"function Foo() {\n list = [];\n}\npush = function(a) {\n list.push(a);\n}\nlast = function() {\n return list.pop();\n}\n"
)
class Interesting(DummyInteresting):
def interesting(sub, conditionArgs, tempPrefix):
with open("a.txt", "rb") as f:
return f.read() in valid_reductions
l = lithium.Lithium()
for testcaseType in (lithium.TestcaseChar, lithium.TestcaseLine,
lithium.TestcaseSymbol):
log.info("Trying with testcase type %s:", testcaseType.__name__)
with open("a.txt", "wb") as f:
f.write(valid_reductions[0])
l.conditionScript = Interesting()
l.strategy = lithium.ReplacePropertiesByGlobals()
l.testcase = testcaseType()
l.testcase.readTestcase("a.txt")
self.assertEqual(l.run(), 0)
with open("a.txt", "rb") as f:
if testcaseType is lithium.TestcaseChar:
self.assertEqual(
f.read(), valid_reductions[0]
) # Char doesn't give this strategy enough to work with
else:
self.assertEqual(f.read(), valid_reductions[-1])
def run(args, classpath, java_file, together_java_files):
"""
Main entry of the file minimization.
@param args args parsed by ArgumentParser, used to running debugged tool
@param classpath the necessary classpath to compile the given java_file
@param java_file the java file that need to be minimized
@param together_java_files files that need to run with the given java_file to trigger tool crash
"""
print("====== Minimizing file {} ======".format(java_file))
FileMinimization.preprocess(java_file)
l = lithium.Lithium()
l.conditionArgs = args
l.conditionScript = FileInterestingJudger(java_file, together_java_files, classpath)
l.testcase = lithium.TestcaseLine()
l.testcase.readTestcase(java_file)
# First round of reduction by main minimization algorithm
print("====== Performing main minimization algorithm ======")
l.strategy = lithium.Minimize()
l.run()
print("------ main minimization algorithm done ------")
# Second round of reduction, focus on reducing balanced pairs
print("====== Minimizing balanced pairs ======")
l.strategy = lithium.MinimizeBalancedPairs()
l.run()
print("------ Minimizing balanced pairs done ------")
# Third round ofreduction, reducing surrounding pairs
print("====== Minimizing surrounding pairs ======")
l.strategy = lithium.MinimizeSurroundingPairs()
l.run()
print("------ Minimizing surrounding pairs done ------")
# Final round of reduction, repeat the main minimization algorithm
print("====== Performing main minimization algorithm ======")
l.strategy = lithium.Minimize()
l.run()
print("------ main minimization algorithm done ------")
print("------ file {} has been minimized ------".format(java_file))
def test_minimize_around(self):
class Interesting(DummyInteresting):
def interesting(sub, conditionArgs, tempPrefix):
with open("a.txt", "rb") as f:
data = f.read()
return b"o\n" in data and len(set(data.split(b"o\n"))) == 1
l = lithium.Lithium()
l.conditionScript = Interesting()
l.strategy = lithium.MinimizeSurroundingPairs()
for testcaseType in (lithium.TestcaseChar, lithium.TestcaseLine,
lithium.TestcaseSymbol):
log.info("Trying with testcase type %s:", testcaseType.__name__)
with open("a.txt", "wb") as f:
f.write(b"x\nx\nx\no\nx\nx\nx\n")
l.testcase = testcaseType()
l.testcase.readTestcase("a.txt")
self.assertEqual(l.run(), 0)
with open("a.txt", "rb") as f:
self.assertEqual(f.read(), b"o\n")
def test_outputs_timeout() -> None:
"""interestingness 'outputs' --timeout test"""
lith = lithium.Lithium()
# check that --timeout works
start_time = time.time()
result = lith.main([
"--strategy",
"check-only",
"--testcase",
"temp.js",
"outputs",
"--timeout",
"1",
"blah",
] + SLEEP_CMD + ["3"])
elapsed = time.time() - start_time
assert result == 1
assert elapsed >= 1
assert lith.test_count == 1
def test_replace_arguments(testcase_cls):
"""test that replace arguments strategy works"""
original = b"function foo(a,b) {\n list = a + b;\n}\nfoo(2,3)\n"
expected = b"function foo() {\n list = a + b;\n}\na = 2;\nb = 3;\nfoo()\n"
valid_reductions = {
original,
b"function foo(a) {\n list = a + b;\n}\nb = 3;\nfoo(2)\n",
b"function foo(a) {\n list = a + b;\n}\nb = 3;\nfoo(2,3)\n",
b"function foo(b) {\n list = a + b;\n}\na = 2;\nfoo(3)\n",
b"function foo() {\n list = a + b;\n}\na = 2;\nb = 3;\nfoo(2,3)\n",
expected,
}
test_path = Path("a.txt")
class _Interesting:
# pylint: disable=missing-function-docstring,no-self-use
def init(self, condition_args):
pass
def interesting(self, *_):
return test_path.read_bytes() in valid_reductions
def cleanup(self, condition_args):
pass
obj = lithium.Lithium()
obj.condition_script = _Interesting()
obj.strategy = lithium.strategies.ReplaceArgumentsByGlobals()
test_path.write_bytes(original)
obj.testcase = testcase_cls()
obj.testcase.load(test_path)
is_char = testcase_cls is lithium.testcases.TestcaseChar
assert obj.run() == int(is_char)
data = test_path.read_bytes()
if is_char:
# Char doesn't give this strategy enough to work with
assert data == original
else:
assert data == expected
def test_empty(self):
l = lithium.Lithium()
with open("empty.txt", "w"):
pass
class Interesting(DummyInteresting):
inter = False
def interesting(sub, conditionArgs, tempPrefix): # pylint: disable=no-self-argument
# pylint: disable=missing-return-doc,missing-return-type-doc
return sub.inter
l.conditionScript = Interesting()
l.strategy = lithium.Minimize()
l.testcase = lithium.TestcaseLine()
l.testcase.readTestcase("empty.txt")
with self.assertLogs("lithium") as logs:
self.assertEqual(l.run(), 1)
self.assertIn("INFO:lithium:Lithium result: the original testcase is not 'interesting'!", logs.output)
Interesting.inter = True
with self.assertLogs("lithium") as logs:
self.assertEqual(l.run(), 0)
self.assertIn("INFO:lithium:The file has 0 lines so there's nothing for Lithium to try to remove!", logs.output)
def test_minimize(testcase_cls):
"""test that minimize strategy works"""
test_path = Path("a.txt")
class _Interesting:
# pylint: disable=missing-function-docstring,no-self-use
def init(self, condition_args):
pass
def interesting(self, *_):
return b"o\n" in test_path.read_bytes()
def cleanup(self, condition_args):
pass
obj = lithium.Lithium()
obj.condition_script = _Interesting()
obj.strategy = lithium.strategies.Minimize()
test_path.write_bytes(b"x\n\nx\nx\no\nx\nx\nx\n")
obj.testcase = testcase_cls()
obj.testcase.load(test_path)
assert obj.run() == 0
assert test_path.read_bytes() == b"o\n"
def test_minimize_around(testcase_cls) -> None:
"""test that minimize around strategy works"""
test_path = Path("a.txt")
class _Interesting:
# pylint: disable=missing-function-docstring,no-self-use
def init(self, condition_args):
pass
def interesting(self, *_):
data = test_path.read_bytes()
return b"o\n" in data and len(set(data.split(b"o\n"))) == 1
def cleanup(self, condition_args):
pass
obj = lithium.Lithium()
obj.condition_script = _Interesting()
obj.strategy = lithium.strategies.MinimizeSurroundingPairs()
test_path.write_bytes(b"x\nx\nx\no\nx\nx\nx\n")
obj.testcase = testcase_cls()
obj.testcase.load(test_path)
assert obj.run() == 0
assert test_path.read_bytes() == b"o\n"
def test_outputs(self):
"""Tests for the 'hangs' interestingness test"""
l = lithium.Lithium()
with open("temp.js", "w"):
pass
# test that `ls temp.js` contains "temp.js"
result = l.main(["outputs", "temp.js"] + self.ls_cmd + ["temp.js"])
self.assertEqual(result, 0)
# test that `ls temp.js` does not contain "blah"
result = l.main(["outputs", "blah"] + self.ls_cmd + ["temp.js"])
self.assertEqual(result, 1)
# check that --timeout works
start_time = time.time()
result = l.main(["--testcase", "temp.js", "outputs", "--timeout", "1", "blah"] + self.sleep_cmd + ["3"])
elapsed = time.time() - start_time
self.assertEqual(result, 1)
self.assertGreaterEqual(elapsed, 1)
# test that regex matches work too
result = l.main(["outputs", "--regex", r"^.*js\s?$"] + self.ls_cmd + ["temp.js"])
self.assertEqual(result, 0)
def run(self, strategies=None):
"""Run reduction.
"""
assert self._testcase is not None
assert self._reporter is not None
try:
# set up lithium
reducer = lithium.Lithium()
self._orig_sig = self._signature
self.landing_page = self._testcase
reducer.conditionScript = LithiumInterestingProxy(self)
# if we created a harness to iterate over history, files_to_reduce is initially just
# that harness
# otherwise, the first stage will be skipped and we will scan for all testcases to
# reduce in the second stage
run_state = RunState([self._testcase])
# resolve list of strategies to apply
reduce_stages = [strategies_module.MinimizeCacheIterHarness, strategies_module.ScanFilesToReduce]
if not self._skip_analysis:
if self._cache_iter_harness_created:
# if we created a cache iterator harness analyze that first
reduce_stages.insert(0, strategies_module.AnalyzeTestcase)
reduce_stages.append(strategies_module.AnalyzeTestcase)
if strategies is None:
strategies = self.DEFAULT_STRATEGIES
strategies_lut = strategies_module.strategies_by_name()
for strat in strategies:
try:
strat = strategies_lut[strat]
except KeyError:
raise ReducerError("Unknown strategy given: %r" % (strat,))
reduce_stages.append(strat)
# run lithium reduce with strategies
files_reduced = 0
for strategy_num, strategy_type in enumerate(reduce_stages):
result = -1
strategy = strategy_type(self, run_state, reducer)
for testcase_path in run_state.files_to_reduce:
strategy.read_testcase(testcase_path)
if strategy.should_skip():
result = 0
continue
self.reduce_file = testcase_path
# set up tempdir manually so it doesn't go in cwd
reducer.tempDir = tempfile.mkdtemp(
prefix="lith-%d-%s" % (strategy_num, strategy_type.name),
dir=self._tmpdir)
reducer.testCount = reducer.testTotal = 0
result = reducer.run()
try:
if result == 0:
strategy.on_success()
files_reduced += 1
else:
strategy.on_failure()
result = 0 # if we passed on failure, don't fail below
except StopIteration:
break
if result != 0:
# reducer failed to repro the crash
if files_reduced == 0:
# first stage, couldn't repro at all
LOG.warning("Could not reduce: The testcase was not reproducible")
self._result_code = FuzzManagerReporter.QUAL_NOT_REPRODUCIBLE
else:
# subsequent stage, reducing broke the testcase?
# unclear how to recover from this.
# just report failure and hopefully we have another to try
LOG.warning("%s failed to reproduce. Previous stage broke the testcase?",
strategy_type.__name__)
self._result_code = FuzzManagerReporter.QUAL_REDUCER_BROKE
return False
# all stages succeeded
reduced_size = run_state.total_size()
if reduced_size == run_state.original_size:
raise ReducerError("Reducer succeeded but nothing was reduced!")
self._report_result(self._best_testcase,
self._interesting_report,
FuzzManagerReporter.QUAL_REDUCED_RESULT,
force=True)
# change original quality so unbucketed crashes don't reduce again
self._result_code = FuzzManagerReporter.QUAL_REDUCED_ORIGINAL
return True
except ReducerError as exc:
LOG.warning("Could not reduce: %s", exc)
self._result_code = FuzzManagerReporter.QUAL_REDUCER_ERROR
return False
except Exception: # pylint: disable=broad-except
LOG.exception("Exception during reduce")
self._result_code = FuzzManagerReporter.QUAL_REDUCER_ERROR
return False
finally:
self._report_other_crashes()
def test_repeat(self):
"""Tests for the 'repeat' interestingness test"""
l = lithium.Lithium() # noqa: E741
with open("temp.js", "w") as tempf:
tempf.write("hello")
# Check for a known string
result = l.main(["repeat", "5", "outputs", "hello"] + self.cat_cmd +
["temp.js"])
self.assertEqual(result, 0)
# Look for a non-existent string, so the "repeat" test tries looping the maximum number of iterations (5x)
with self.assertLogs("lithium") as test_logs:
result = l.main(["repeat", "5", "outputs", "notfound"] +
self.cat_cmd + ["temp.js"])
self.assertEqual(result, 1)
found_count = 0
last_count = 0
# scan the log output to see how many tests were performed
for rec in test_logs.records:
message = rec.getMessage()
if "Repeat number " in message:
found_count += 1
last_count = rec.args[0]
self.assertEqual(found_count, 5) # Should have run 5x
self.assertEqual(
found_count,
last_count) # We should have identical count outputs
# Check that replacements on the CLI work properly
# Lower boundary - check that 0 (just outside [1]) is not found
with open("temp1a.js", "w") as tempf1a:
tempf1a.write("num0")
result = l.main(
["repeat", "1", "outputs", "--timeout=9", "numREPEATNUM"] +
self.cat_cmd + ["temp1a.js"])
self.assertEqual(result, 1)
# Upper boundary - check that 2 (just outside [1]) is not found
with open("temp1b.js", "w") as tempf1b:
tempf1b.write("num2")
result = l.main(
["repeat", "1", "outputs", "--timeout=9", "numREPEATNUM"] +
self.cat_cmd + ["temp1b.js"])
self.assertEqual(result, 1)
# Lower boundary - check that 0 (just outside [1,2]) is not found
with open("temp2a.js", "w") as tempf2a:
tempf2a.write("num0")
result = l.main(
["repeat", "2", "outputs", "--timeout=9", "numREPEATNUM"] +
self.cat_cmd + ["temp2a.js"])
self.assertEqual(result, 1)
# Upper boundary - check that 3 (just outside [1,2]) is not found
with open("temp2b.js", "w") as tempf2b:
tempf2b.write("num3")
result = l.main(
["repeat", "2", "outputs", "--timeout=9", "numREPEATNUM"] +
self.cat_cmd + ["temp2b.js"])
self.assertEqual(result, 1)
def test_executable(self):
with self.assertRaisesRegex(SystemExit, "0"):
lithium.Lithium().main(["-h"])
def test_executable():
"""test lithium main help call"""
with pytest.raises(SystemExit, match="0"):
lithium.Lithium().main(["-h"])
def test_replace_properties(testcase_cls):
"""test that replace properties strategy works"""
original = (
# original: this.list, prototype.push, prototype.last
b"function Foo() {\n this.list = [];\n}\n"
b"Foo.prototype.push = function(a) {\n this.list.push(a);\n}\n"
b"Foo.prototype.last = function() {\n return this.list.pop();\n}\n")
expected = (
# reduced: list, push, last
b"function Foo() {\n list = [];\n}\n"
b"push = function(a) {\n list.push(a);\n}\n"
b"last = function() {\n return list.pop();\n}\n")
valid_reductions = {
original,
# this.list, prototype.push, last
b"function Foo() {\n this.list = [];\n}\n"
b"Foo.prototype.push = function(a) {\n this.list.push(a);\n}\n"
b"last = function() {\n return this.list.pop();\n}\n",
# this.list, push, prototype.last
b"function Foo() {\n this.list = [];\n}\n"
b"push = function(a) {\n this.list.push(a);\n}\n"
b"Foo.prototype.last = function() {\n return this.list.pop();\n}\n",
# this.list, push, last
b"function Foo() {\n this.list = [];\n}\n"
b"push = function(a) {\n this.list.push(a);\n}\n"
b"last = function() {\n return this.list.pop();\n}\n",
# list, prototype.push, prototype.last
b"function Foo() {\n list = [];\n}\n"
b"Foo.prototype.push = function(a) {\n list.push(a);\n}\n"
b"Foo.prototype.last = function() {\n return list.pop();\n}\n",
# list, prototype.push, last
b"function Foo() {\n list = [];\n}\n"
b"Foo.prototype.push = function(a) {\n list.push(a);\n}\n"
b"last = function() {\n return list.pop();\n}\n",
# list, push, prototype.last
b"function Foo() {\n list = [];\n}\n"
b"push = function(a) {\n list.push(a);\n}\n"
b"Foo.prototype.last = function() {\n return list.pop();\n}\n",
expected,
}
test_path = Path("a.txt")
class _Interesting:
# pylint: disable=missing-function-docstring,no-self-use
def init(self, condition_args):
pass
def interesting(self, *_):
return test_path.read_bytes() in valid_reductions
def cleanup(self, condition_args):
pass
obj = lithium.Lithium()
test_path.write_bytes(original)
obj.condition_script = _Interesting()
obj.strategy = lithium.strategies.ReplacePropertiesByGlobals()
obj.testcase = testcase_cls()
obj.testcase.load(test_path)
is_char = testcase_cls is lithium.testcases.TestcaseChar
assert obj.run() == int(is_char)
data = test_path.read_bytes()
if is_char:
# Char doesn't give this strategy enough to work with
assert data == original
else:
assert data == expected
