def testRetest(self):
session_file_name = 'testSessionResume.session'
try:
os.remove(session_file_name)
except:
pass
self.logger.info(
'Make the fuzzer stop after 2 tests by reaching max failures')
target_config = {
'1': {
'send': {
"raise exception": True
}
},
'3': {
'send': {
"raise exception": True
}
},
'5': {
'send': {
"raise exception": True
}
},
}
self.target = ServerTargetMock(target_config, logger=self.logger)
cmd_line = '--test-list=0-10 --session=%s' % (session_file_name)
self.fuzzer = ServerFuzzer(name='TestServerFuzzer',
logger=self.logger,
option_line=cmd_line)
self.fuzzer.set_interface(self.interface)
self.fuzzer.set_model(self.new_model())
self.fuzzer.set_target(self.target)
self.fuzzer.start()
pre_test_list = self.target.instrument.list_get('pre_test')
self.assertListEqual(pre_test_list, list(range(-1, 11)))
self.fuzzer.stop()
self.logger.info('Now use the same session file to rerun failed tests')
cmd_line = '--retest=%s --no-env-test' % (session_file_name)
self.target = ServerTargetMock({}, logger=self.logger)
self.fuzzer = ServerFuzzer(name='TestServerFuzzer',
logger=self.logger,
option_line=cmd_line)
self.fuzzer.set_interface(self.interface)
self.fuzzer.set_model(self.new_model())
self.fuzzer.set_target(self.target)
self.fuzzer.start()
pre_test_list = self.target.instrument.list_get('pre_test')
self.assertListEqual(pre_test_list, [1, 3, 5])
os.remove(session_file_name)
def testAllFailedTestsHaveReports(self):
config = {
'10': {
'report': {
'status': 'failed',
'reason': 'failure reason'
}
},
'11': {
'report': {
'status': 'failed',
'reason': 'failure reason'
}
},
'12': {
'report': {
'status': 'failed',
'reason': 'failure reason'
}
},
'13': {
'report': {
'status': 'failed',
'reason': 'failure reason'
}
}
}
target = ServerTargetMock(config, logger=self.logger)
self.fuzzer.set_target(target)
self.fuzzer.start()
info = self.fuzzer._get_session_info()
reports = self.fuzzer.dataman.get_report_test_ids()
self.assertEqual(reports, sorted([int(x) for x in config.keys()]))
self.assertEqual(info.failure_count, len(config))
def testMaxFailures(self):
target_config = {
'1': {
'send': {
"raise exception": True
}
},
'2': {
'send': {
"raise exception": True
}
},
'3': {
'send': {
"raise exception": True
}
},
}
self.target = ServerTargetMock(target_config, logger=self.logger)
cmd_line = '--test-list=0-10'
self.fuzzer = ServerFuzzer(name='TestServerFuzzer',
logger=self.logger,
option_line=cmd_line)
self.fuzzer.set_interface(self.interface)
self.fuzzer.set_model(self.model)
self.fuzzer.set_target(self.target)
self.fuzzer.set_max_failures(3)
self.fuzzer.start()
# check what tests were started by the fuzzer
pre_test_list = self.target.instrument.list_get('pre_test')
self.assertListEqual(pre_test_list, [-1, 0, 1, 2, 3])
def testTestFailedWhenReportIsFailed(self):
config = {'13': {'report': {'failed': True}}}
target = ServerTargetMock(config, logger=self.logger)
self.fuzzer.set_target(target)
self.fuzzer.start()
info = self.fuzzer._get_session_info()
reports = self.fuzzer.dataman.get_report_test_ids()
self.assertEqual(reports, [int(x) for x in config.keys()])
self.assertEqual(info.failure_count, len(config))
def _MOVE_TO_TARGET_TESTS_test_send_failure(self):
config = {'12': {'send': {"raise exception": True}}}
send_error_target = ServerTargetMock(config, logger=self.logger)
self.fuzzer.set_target(send_error_target)
self.fuzzer.start()
info = self.fuzzer._get_session_info()
reports = self.fuzzer._get_reports_manager()
self.assertEqual(len(reports), 1)
self.assertTrue(12 in reports)
self.assertEqual(info.failure_count, 1)
def _runFuzzerWithReportList(self, uut, report_list):
config = {}
for report_id in report_list:
config[str(report_id)] = {'report': {'status': 'failed', 'reason': 'failure reason'}}
self.fuzzer.set_interface(uut)
target = ServerTargetMock(config, logger=self.logger)
self.fuzzer.set_target(target)
self.fuzzer.start()
def testStoringAllReportsWhenStoreAllReportsIsSetToTrue(self):
config = {}
target = ServerTargetMock(config, logger=self.logger)
self.fuzzer.set_store_all_reports(True)
self.fuzzer.set_target(target)
self.fuzzer.start()
info = self.fuzzer._get_session_info()
reports = self.fuzzer.dataman.get_report_test_ids()
expected_mutation_count = self.end_index - self.start_index + 1
expected_failure_count = 0
self.assertEqual(len(reports), expected_mutation_count)
self.assertEqual(info.failure_count, expected_failure_count)
def prepare(self):
self.start_index = 10
self.end_index = 20
self.delay_duration = 0
self.session_file_name = None
self.interface = EmptyInterface()
self.model = self.new_model()
self.target = ServerTargetMock({}, logger=self.logger)
self.fuzzer = ServerFuzzer(name="TestServerFuzzer", logger=self.logger)
self.fuzzer.set_interface(self.interface)
self.fuzzer.set_model(self.model)
self.fuzzer.set_target(self.target)
self.fuzzer.set_range(self.start_index, self.end_index)
self.fuzzer.set_delay_between_tests(self.delay_duration)