class TestSchedulerCompare(unittest.TestCase):
def setUp(self):
self.config = ConfigParser()
self.config.add_section('ExpertSettings')
self.config.set('ExpertSettings', 'block_delay', '2')
self.config.set('ExpertSettings', 'ssdeep_ignore', '80')
self.bs_patch = unittest.mock.patch(
target='storage.binary_service.BinaryService',
new=lambda _: MockDbInterface)
self.bs_patch.start()
self.compare_scheduler = CompareScheduler(
config=self.config,
db_interface=MockDbInterface(config=self.config),
testing=True)
def tearDown(self):
self.compare_scheduler.shutdown()
self.bs_patch.stop()
gc.collect()
def test_start_compare(self):
result = self.compare_scheduler.add_task(('existing_id', True))
self.assertIsNone(result, 'result ist not None')
uid, redo = self.compare_scheduler.in_queue.get(timeout=2)
self.assertEqual(uid, 'existing_id', 'retrieved id not correct')
self.assertTrue(redo, 'redo argument not correct')
def test_start(self):
self.compare_scheduler.start()
sleep(2)
def test_compare_single_run(self):
compares_done = set()
self.compare_scheduler.in_queue.put(
(self.compare_scheduler.db_interface.test_object.get_uid(), False))
self.compare_scheduler._compare_single_run(compares_done)
self.assertEqual(len(compares_done), 1,
'compares done not set correct')
self.assertIn(
self.compare_scheduler.db_interface.test_object.get_uid(),
compares_done, 'correct uid not in compares done')
def test_decide_whether_to_process(self):
compares_done = set('a')
self.assertTrue(
self.compare_scheduler._decide_whether_to_process(
'b', False, compares_done),
'none existing should always be done')
self.assertTrue(
self.compare_scheduler._decide_whether_to_process(
'a', True, compares_done),
'redo is true so result should be true')
self.assertFalse(
self.compare_scheduler._decide_whether_to_process(
'a', False, compares_done),
'already done and redo no -> should be false')
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(
config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'files_in_common': {
'all': [],
'collapse': False
}
},
'Software': {
'Compare Skipped': {
'all':
'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(config=self._config, post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_object(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value > 7:
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(file_path='{}/container/test.7z'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=10)
compare_id = unify_string_list(';'.join([fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertFalse(isinstance(result, str), 'compare result should exist')
self.assertEqual(result['plugins']['Software'], self._expected_result()['Software'])
self.assertCountEqual(result['plugins']['File_Coverage']['exclusive_files'], self._expected_result()['File_Coverage']['exclusive_files'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'exclusive_files': {
'418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787': [],
'd38970f8c5153d1041810d0908292bc8df21e7fd88aab211a8fb96c54afe6b01_319': [],
'collapse': False
},
'files_in_common': {
'all': [
'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28',
'289b5a050a83837f192d7129e4c4e02570b94b4924e50159fad5ed1067cfbfeb_20',
'd558c9339cb967341d701e3184f863d3928973fccdc1d96042583730b5c7b76a_62'
],
'collapse': False
},
'similar_files': {}
},
'Software': {
'Compare Skipped': {
'all': 'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}