def test_diff_test(self): """Tests for the 'diff_test' interestingness test""" l = lithium.Lithium() # noqa: E741 with open("temp.js", "w"): pass # test that the parameters "-a" and "-b" of diff_test work result = l.main([ "diff_test", "--timeout", "99", "-a", "flags_one", "-b", "flags_two_a flags_two_b" ] + self.ls_cmd + ["temp.js"]) self.assertEqual(result, 0) result = l.main([ "diff_test", "--a-args", "flags_one_a flags_one_b", "--b-args", "flags_two" ] + self.ls_cmd + ["temp.js"]) self.assertEqual(result, 0)
def test_interestingness_outputs_multiline(capsys, pattern, expected): """Tests for the 'outputs' interestingness test with multiline pattern""" lith = lithium.Lithium() with open("temp.js", "wb") as tmp_f: tmp_f.write(b"line A\nline B\nline C\nline D\nline E\n") capsys.readouterr() # clear captured output buffers result = lith.main([ "outputs", pattern, ] + CAT_CMD + ["temp.js"]) assert result == 0, "%r not found in %r" % (pattern, open("temp.js").read()) # assert lith.test_count == 1 captured = capsys.readouterr() assert "[Found string in: %r]" % (expected, ) in captured.out
def test_crashes_2(examples_path: Path) -> None: """crash test for the 'crashes' interestingness test""" lith = lithium.Lithium() # if a compiler is available, compile a simple crashing test program src = examples_path / "crash.c" exe = Path.cwd().resolve() / ("crash.exe" if platform.system() == "Windows" else "crash") try: _compile(src, exe) except RuntimeError as exc: LOG.warning(exc) pytest.skip("compile 'crash.c' failed") result = lith.main( ["--strategy", "check-only", "crashes", str(exe), "temp.js"]) assert result == 0 assert lith.test_count == 1
def test_diff_test_0() -> None: """test for the 'diff_test' interestingness test""" lith = lithium.Lithium() # test that the parameters "-a" and "-b" of diff_test work result = lith.main([ "--strategy", "check-only", "diff_test", "--timeout", "99", "-a", "flags_one", "-b", "flags_two_a flags_two_b", ] + LS_CMD + ["temp.js"]) assert result == 0 assert lith.test_count == 1
def test_minimize(self): class Interesting(DummyInteresting): def interesting(sub, conditionArgs, tempPrefix): # pylint: disable=no-self-argument # pylint: disable=missing-return-doc,missing-return-type-doc with open("a.txt", "rb") as f: return b"o\n" in f.read() l = lithium.Lithium() l.conditionScript = Interesting() l.strategy = lithium.Minimize() for testcaseType in (lithium.TestcaseChar, lithium.TestcaseLine, lithium.TestcaseSymbol): log.info("Trying with testcase type %s:", testcaseType.__name__) with open("a.txt", "wb") as f: f.write(b"x\n\nx\nx\no\nx\nx\nx\n") l.testcase = testcaseType() l.testcase.readTestcase("a.txt") self.assertEqual(l.run(), 0) with open("a.txt", "rb") as f: self.assertEqual(f.read(), b"o\n")
def test_crashes_1() -> None: """timeout test for the 'crashes' interestingness test""" lith = lithium.Lithium() # check that --timeout works start_time = time.time() result = lith.main([ "--strategy", "check-only", "--testcase", "temp.js", "crashes", "--timeout", "1", ] + SLEEP_CMD + ["3"]) elapsed = time.time() - start_time assert result == 1 assert elapsed >= 1 assert lith.test_count == 1
def test_replace_properties(self): valid_reductions = ( # original: this.list, prototype.push, prototype.last b"function Foo() {\n this.list = [];\n}\nFoo.prototype.push = function(a) {\n this.list.push(a);\n}\nFoo.prototype.last = function() {\n return this.list.pop();\n}\n", # this.list, prototype.push, last b"function Foo() {\n this.list = [];\n}\nFoo.prototype.push = function(a) {\n this.list.push(a);\n}\nlast = function() {\n return this.list.pop();\n}\n", # this.list, push, prototype.last b"function Foo() {\n this.list = [];\n}\npush = function(a) {\n this.list.push(a);\n}\nFoo.prototype.last = function() {\n return this.list.pop();\n}\n", # this.list, push, last b"function Foo() {\n this.list = [];\n}\npush = function(a) {\n this.list.push(a);\n}\nlast = function() {\n return this.list.pop();\n}\n", # list, prototype.push, prototype.last b"function Foo() {\n list = [];\n}\nFoo.prototype.push = function(a) {\n list.push(a);\n}\nFoo.prototype.last = function() {\n return list.pop();\n}\n", # list, prototype.push, last b"function Foo() {\n list = [];\n}\nFoo.prototype.push = function(a) {\n list.push(a);\n}\nlast = function() {\n return list.pop();\n}\n", # list, push, prototype.last b"function Foo() {\n list = [];\n}\npush = function(a) {\n list.push(a);\n}\nFoo.prototype.last = function() {\n return list.pop();\n}\n", # reduced: list, push, last b"function Foo() {\n list = [];\n}\npush = function(a) {\n list.push(a);\n}\nlast = function() {\n return list.pop();\n}\n" ) class Interesting(DummyInteresting): def interesting(sub, conditionArgs, tempPrefix): with open("a.txt", "rb") as f: return f.read() in valid_reductions l = lithium.Lithium() for testcaseType in (lithium.TestcaseChar, lithium.TestcaseLine, lithium.TestcaseSymbol): log.info("Trying with testcase type %s:", testcaseType.__name__) with open("a.txt", "wb") as f: f.write(valid_reductions[0]) l.conditionScript = Interesting() l.strategy = lithium.ReplacePropertiesByGlobals() l.testcase = testcaseType() l.testcase.readTestcase("a.txt") self.assertEqual(l.run(), 0) with open("a.txt", "rb") as f: if testcaseType is lithium.TestcaseChar: self.assertEqual( f.read(), valid_reductions[0] ) # Char doesn't give this strategy enough to work with else: self.assertEqual(f.read(), valid_reductions[-1])
def run(args, classpath, java_file, together_java_files): """ Main entry of the file minimization. @param args args parsed by ArgumentParser, used to running debugged tool @param classpath the necessary classpath to compile the given java_file @param java_file the java file that need to be minimized @param together_java_files files that need to run with the given java_file to trigger tool crash """ print("====== Minimizing file {} ======".format(java_file)) FileMinimization.preprocess(java_file) l = lithium.Lithium() l.conditionArgs = args l.conditionScript = FileInterestingJudger(java_file, together_java_files, classpath) l.testcase = lithium.TestcaseLine() l.testcase.readTestcase(java_file) # First round of reduction by main minimization algorithm print("====== Performing main minimization algorithm ======") l.strategy = lithium.Minimize() l.run() print("------ main minimization algorithm done ------") # Second round of reduction, focus on reducing balanced pairs print("====== Minimizing balanced pairs ======") l.strategy = lithium.MinimizeBalancedPairs() l.run() print("------ Minimizing balanced pairs done ------") # Third round ofreduction, reducing surrounding pairs print("====== Minimizing surrounding pairs ======") l.strategy = lithium.MinimizeSurroundingPairs() l.run() print("------ Minimizing surrounding pairs done ------") # Final round of reduction, repeat the main minimization algorithm print("====== Performing main minimization algorithm ======") l.strategy = lithium.Minimize() l.run() print("------ main minimization algorithm done ------") print("------ file {} has been minimized ------".format(java_file))
def test_minimize_around(self): class Interesting(DummyInteresting): def interesting(sub, conditionArgs, tempPrefix): with open("a.txt", "rb") as f: data = f.read() return b"o\n" in data and len(set(data.split(b"o\n"))) == 1 l = lithium.Lithium() l.conditionScript = Interesting() l.strategy = lithium.MinimizeSurroundingPairs() for testcaseType in (lithium.TestcaseChar, lithium.TestcaseLine, lithium.TestcaseSymbol): log.info("Trying with testcase type %s:", testcaseType.__name__) with open("a.txt", "wb") as f: f.write(b"x\nx\nx\no\nx\nx\nx\n") l.testcase = testcaseType() l.testcase.readTestcase("a.txt") self.assertEqual(l.run(), 0) with open("a.txt", "rb") as f: self.assertEqual(f.read(), b"o\n")
def test_outputs_timeout() -> None: """interestingness 'outputs' --timeout test""" lith = lithium.Lithium() # check that --timeout works start_time = time.time() result = lith.main([ "--strategy", "check-only", "--testcase", "temp.js", "outputs", "--timeout", "1", "blah", ] + SLEEP_CMD + ["3"]) elapsed = time.time() - start_time assert result == 1 assert elapsed >= 1 assert lith.test_count == 1
def test_replace_arguments(testcase_cls): """test that replace arguments strategy works""" original = b"function foo(a,b) {\n list = a + b;\n}\nfoo(2,3)\n" expected = b"function foo() {\n list = a + b;\n}\na = 2;\nb = 3;\nfoo()\n" valid_reductions = { original, b"function foo(a) {\n list = a + b;\n}\nb = 3;\nfoo(2)\n", b"function foo(a) {\n list = a + b;\n}\nb = 3;\nfoo(2,3)\n", b"function foo(b) {\n list = a + b;\n}\na = 2;\nfoo(3)\n", b"function foo() {\n list = a + b;\n}\na = 2;\nb = 3;\nfoo(2,3)\n", expected, } test_path = Path("a.txt") class _Interesting: # pylint: disable=missing-function-docstring,no-self-use def init(self, condition_args): pass def interesting(self, *_): return test_path.read_bytes() in valid_reductions def cleanup(self, condition_args): pass obj = lithium.Lithium() obj.condition_script = _Interesting() obj.strategy = lithium.strategies.ReplaceArgumentsByGlobals() test_path.write_bytes(original) obj.testcase = testcase_cls() obj.testcase.load(test_path) is_char = testcase_cls is lithium.testcases.TestcaseChar assert obj.run() == int(is_char) data = test_path.read_bytes() if is_char: # Char doesn't give this strategy enough to work with assert data == original else: assert data == expected
def test_empty(self): l = lithium.Lithium() with open("empty.txt", "w"): pass class Interesting(DummyInteresting): inter = False def interesting(sub, conditionArgs, tempPrefix): # pylint: disable=no-self-argument # pylint: disable=missing-return-doc,missing-return-type-doc return sub.inter l.conditionScript = Interesting() l.strategy = lithium.Minimize() l.testcase = lithium.TestcaseLine() l.testcase.readTestcase("empty.txt") with self.assertLogs("lithium") as logs: self.assertEqual(l.run(), 1) self.assertIn("INFO:lithium:Lithium result: the original testcase is not 'interesting'!", logs.output) Interesting.inter = True with self.assertLogs("lithium") as logs: self.assertEqual(l.run(), 0) self.assertIn("INFO:lithium:The file has 0 lines so there's nothing for Lithium to try to remove!", logs.output)
def test_minimize(testcase_cls): """test that minimize strategy works""" test_path = Path("a.txt") class _Interesting: # pylint: disable=missing-function-docstring,no-self-use def init(self, condition_args): pass def interesting(self, *_): return b"o\n" in test_path.read_bytes() def cleanup(self, condition_args): pass obj = lithium.Lithium() obj.condition_script = _Interesting() obj.strategy = lithium.strategies.Minimize() test_path.write_bytes(b"x\n\nx\nx\no\nx\nx\nx\n") obj.testcase = testcase_cls() obj.testcase.load(test_path) assert obj.run() == 0 assert test_path.read_bytes() == b"o\n"
def test_minimize_around(testcase_cls) -> None: """test that minimize around strategy works""" test_path = Path("a.txt") class _Interesting: # pylint: disable=missing-function-docstring,no-self-use def init(self, condition_args): pass def interesting(self, *_): data = test_path.read_bytes() return b"o\n" in data and len(set(data.split(b"o\n"))) == 1 def cleanup(self, condition_args): pass obj = lithium.Lithium() obj.condition_script = _Interesting() obj.strategy = lithium.strategies.MinimizeSurroundingPairs() test_path.write_bytes(b"x\nx\nx\no\nx\nx\nx\n") obj.testcase = testcase_cls() obj.testcase.load(test_path) assert obj.run() == 0 assert test_path.read_bytes() == b"o\n"
def test_outputs(self): """Tests for the 'hangs' interestingness test""" l = lithium.Lithium() with open("temp.js", "w"): pass # test that `ls temp.js` contains "temp.js" result = l.main(["outputs", "temp.js"] + self.ls_cmd + ["temp.js"]) self.assertEqual(result, 0) # test that `ls temp.js` does not contain "blah" result = l.main(["outputs", "blah"] + self.ls_cmd + ["temp.js"]) self.assertEqual(result, 1) # check that --timeout works start_time = time.time() result = l.main(["--testcase", "temp.js", "outputs", "--timeout", "1", "blah"] + self.sleep_cmd + ["3"]) elapsed = time.time() - start_time self.assertEqual(result, 1) self.assertGreaterEqual(elapsed, 1) # test that regex matches work too result = l.main(["outputs", "--regex", r"^.*js\s?$"] + self.ls_cmd + ["temp.js"]) self.assertEqual(result, 0)
def run(self, strategies=None): """Run reduction. """ assert self._testcase is not None assert self._reporter is not None try: # set up lithium reducer = lithium.Lithium() self._orig_sig = self._signature self.landing_page = self._testcase reducer.conditionScript = LithiumInterestingProxy(self) # if we created a harness to iterate over history, files_to_reduce is initially just # that harness # otherwise, the first stage will be skipped and we will scan for all testcases to # reduce in the second stage run_state = RunState([self._testcase]) # resolve list of strategies to apply reduce_stages = [strategies_module.MinimizeCacheIterHarness, strategies_module.ScanFilesToReduce] if not self._skip_analysis: if self._cache_iter_harness_created: # if we created a cache iterator harness analyze that first reduce_stages.insert(0, strategies_module.AnalyzeTestcase) reduce_stages.append(strategies_module.AnalyzeTestcase) if strategies is None: strategies = self.DEFAULT_STRATEGIES strategies_lut = strategies_module.strategies_by_name() for strat in strategies: try: strat = strategies_lut[strat] except KeyError: raise ReducerError("Unknown strategy given: %r" % (strat,)) reduce_stages.append(strat) # run lithium reduce with strategies files_reduced = 0 for strategy_num, strategy_type in enumerate(reduce_stages): result = -1 strategy = strategy_type(self, run_state, reducer) for testcase_path in run_state.files_to_reduce: strategy.read_testcase(testcase_path) if strategy.should_skip(): result = 0 continue self.reduce_file = testcase_path # set up tempdir manually so it doesn't go in cwd reducer.tempDir = tempfile.mkdtemp( prefix="lith-%d-%s" % (strategy_num, strategy_type.name), dir=self._tmpdir) reducer.testCount = reducer.testTotal = 0 result = reducer.run() try: if result == 0: strategy.on_success() files_reduced += 1 else: strategy.on_failure() result = 0 # if we passed on failure, don't fail below except StopIteration: break if result != 0: # reducer failed to repro the crash if files_reduced == 0: # first stage, couldn't repro at all LOG.warning("Could not reduce: The testcase was not reproducible") self._result_code = FuzzManagerReporter.QUAL_NOT_REPRODUCIBLE else: # subsequent stage, reducing broke the testcase? # unclear how to recover from this. # just report failure and hopefully we have another to try LOG.warning("%s failed to reproduce. Previous stage broke the testcase?", strategy_type.__name__) self._result_code = FuzzManagerReporter.QUAL_REDUCER_BROKE return False # all stages succeeded reduced_size = run_state.total_size() if reduced_size == run_state.original_size: raise ReducerError("Reducer succeeded but nothing was reduced!") self._report_result(self._best_testcase, self._interesting_report, FuzzManagerReporter.QUAL_REDUCED_RESULT, force=True) # change original quality so unbucketed crashes don't reduce again self._result_code = FuzzManagerReporter.QUAL_REDUCED_ORIGINAL return True except ReducerError as exc: LOG.warning("Could not reduce: %s", exc) self._result_code = FuzzManagerReporter.QUAL_REDUCER_ERROR return False except Exception: # pylint: disable=broad-except LOG.exception("Exception during reduce") self._result_code = FuzzManagerReporter.QUAL_REDUCER_ERROR return False finally: self._report_other_crashes()
def test_repeat(self): """Tests for the 'repeat' interestingness test""" l = lithium.Lithium() # noqa: E741 with open("temp.js", "w") as tempf: tempf.write("hello") # Check for a known string result = l.main(["repeat", "5", "outputs", "hello"] + self.cat_cmd + ["temp.js"]) self.assertEqual(result, 0) # Look for a non-existent string, so the "repeat" test tries looping the maximum number of iterations (5x) with self.assertLogs("lithium") as test_logs: result = l.main(["repeat", "5", "outputs", "notfound"] + self.cat_cmd + ["temp.js"]) self.assertEqual(result, 1) found_count = 0 last_count = 0 # scan the log output to see how many tests were performed for rec in test_logs.records: message = rec.getMessage() if "Repeat number " in message: found_count += 1 last_count = rec.args[0] self.assertEqual(found_count, 5) # Should have run 5x self.assertEqual( found_count, last_count) # We should have identical count outputs # Check that replacements on the CLI work properly # Lower boundary - check that 0 (just outside [1]) is not found with open("temp1a.js", "w") as tempf1a: tempf1a.write("num0") result = l.main( ["repeat", "1", "outputs", "--timeout=9", "numREPEATNUM"] + self.cat_cmd + ["temp1a.js"]) self.assertEqual(result, 1) # Upper boundary - check that 2 (just outside [1]) is not found with open("temp1b.js", "w") as tempf1b: tempf1b.write("num2") result = l.main( ["repeat", "1", "outputs", "--timeout=9", "numREPEATNUM"] + self.cat_cmd + ["temp1b.js"]) self.assertEqual(result, 1) # Lower boundary - check that 0 (just outside [1,2]) is not found with open("temp2a.js", "w") as tempf2a: tempf2a.write("num0") result = l.main( ["repeat", "2", "outputs", "--timeout=9", "numREPEATNUM"] + self.cat_cmd + ["temp2a.js"]) self.assertEqual(result, 1) # Upper boundary - check that 3 (just outside [1,2]) is not found with open("temp2b.js", "w") as tempf2b: tempf2b.write("num3") result = l.main( ["repeat", "2", "outputs", "--timeout=9", "numREPEATNUM"] + self.cat_cmd + ["temp2b.js"]) self.assertEqual(result, 1)
def test_executable(self): with self.assertRaisesRegex(SystemExit, "0"): lithium.Lithium().main(["-h"])
def test_executable(): """test lithium main help call""" with pytest.raises(SystemExit, match="0"): lithium.Lithium().main(["-h"])
def test_replace_properties(testcase_cls): """test that replace properties strategy works""" original = ( # original: this.list, prototype.push, prototype.last b"function Foo() {\n this.list = [];\n}\n" b"Foo.prototype.push = function(a) {\n this.list.push(a);\n}\n" b"Foo.prototype.last = function() {\n return this.list.pop();\n}\n") expected = ( # reduced: list, push, last b"function Foo() {\n list = [];\n}\n" b"push = function(a) {\n list.push(a);\n}\n" b"last = function() {\n return list.pop();\n}\n") valid_reductions = { original, # this.list, prototype.push, last b"function Foo() {\n this.list = [];\n}\n" b"Foo.prototype.push = function(a) {\n this.list.push(a);\n}\n" b"last = function() {\n return this.list.pop();\n}\n", # this.list, push, prototype.last b"function Foo() {\n this.list = [];\n}\n" b"push = function(a) {\n this.list.push(a);\n}\n" b"Foo.prototype.last = function() {\n return this.list.pop();\n}\n", # this.list, push, last b"function Foo() {\n this.list = [];\n}\n" b"push = function(a) {\n this.list.push(a);\n}\n" b"last = function() {\n return this.list.pop();\n}\n", # list, prototype.push, prototype.last b"function Foo() {\n list = [];\n}\n" b"Foo.prototype.push = function(a) {\n list.push(a);\n}\n" b"Foo.prototype.last = function() {\n return list.pop();\n}\n", # list, prototype.push, last b"function Foo() {\n list = [];\n}\n" b"Foo.prototype.push = function(a) {\n list.push(a);\n}\n" b"last = function() {\n return list.pop();\n}\n", # list, push, prototype.last b"function Foo() {\n list = [];\n}\n" b"push = function(a) {\n list.push(a);\n}\n" b"Foo.prototype.last = function() {\n return list.pop();\n}\n", expected, } test_path = Path("a.txt") class _Interesting: # pylint: disable=missing-function-docstring,no-self-use def init(self, condition_args): pass def interesting(self, *_): return test_path.read_bytes() in valid_reductions def cleanup(self, condition_args): pass obj = lithium.Lithium() test_path.write_bytes(original) obj.condition_script = _Interesting() obj.strategy = lithium.strategies.ReplacePropertiesByGlobals() obj.testcase = testcase_cls() obj.testcase.load(test_path) is_char = testcase_cls is lithium.testcases.TestcaseChar assert obj.run() == int(is_char) data = test_path.read_bytes() if is_char: # Char doesn't give this strategy enough to work with assert data == original else: assert data == expected