class TestSchedulerCompare(unittest.TestCase):
def setUp(self):
self.config = ConfigParser()
self.config.add_section('ExpertSettings')
self.config.set('ExpertSettings', 'block_delay', '2')
self.config.set('ExpertSettings', 'ssdeep_ignore', '80')
self.bs_patch = unittest.mock.patch(
target='storage.binary_service.BinaryService',
new=lambda _: MockDbInterface)
self.bs_patch.start()
self.compare_scheduler = CompareScheduler(
config=self.config,
db_interface=MockDbInterface(config=self.config),
testing=True)
def tearDown(self):
self.compare_scheduler.shutdown()
self.bs_patch.stop()
gc.collect()
def test_start_compare(self):
result = self.compare_scheduler.add_task(('existing_id', True))
self.assertIsNone(result, 'result ist not None')
uid, redo = self.compare_scheduler.in_queue.get(timeout=2)
self.assertEqual(uid, 'existing_id', 'retrieved id not correct')
self.assertTrue(redo, 'redo argument not correct')
def test_start(self):
self.compare_scheduler.start()
sleep(2)
def test_compare_single_run(self):
compares_done = set()
self.compare_scheduler.in_queue.put(
(self.compare_scheduler.db_interface.test_object.get_uid(), False))
self.compare_scheduler._compare_single_run(compares_done)
self.assertEqual(len(compares_done), 1,
'compares done not set correct')
self.assertIn(
self.compare_scheduler.db_interface.test_object.get_uid(),
compares_done, 'correct uid not in compares done')
def test_decide_whether_to_process(self):
compares_done = set('a')
self.assertTrue(
self.compare_scheduler._decide_whether_to_process(
'b', False, compares_done),
'none existing should always be done')
self.assertTrue(
self.compare_scheduler._decide_whether_to_process(
'a', True, compares_done),
'redo is true so result should be true')
self.assertFalse(
self.compare_scheduler._decide_whether_to_process(
'a', False, compares_done),
'already done and redo no -> should be false')
class FactBackend(FactBase):
PROGRAM_NAME = 'FACT Backend'
PROGRAM_DESCRIPTION = 'Firmware Analysis and Compare Tool (FACT) Backend'
COMPONENT = 'backend'
def __init__(self):
super().__init__()
try:
self.analysis_service = AnalysisScheduler(config=self.config)
except PluginInitException as error:
logging.critical(
f'Error during initialization of plugin {error.plugin.NAME}. Shutting down FACT backend'
)
complete_shutdown()
self.tagging_service = TaggingDaemon(
analysis_scheduler=self.analysis_service)
self.unpacking_service = UnpackingScheduler(
config=self.config,
post_unpack=self.analysis_service.start_analysis_of_object,
analysis_workload=self.analysis_service.get_scheduled_workload)
self.compare_service = CompareScheduler(config=self.config)
self.intercom = InterComBackEndBinding(
config=self.config,
analysis_service=self.analysis_service,
compare_service=self.compare_service,
unpacking_service=self.unpacking_service)
def main(self):
while self.run:
self.work_load_stat.update(
unpacking_workload=self.unpacking_service.
get_scheduled_workload(),
analysis_workload=self.analysis_service.get_scheduled_workload(
))
if self._exception_occurred():
break
sleep(5)
if self.args.testing:
break
self.shutdown()
def shutdown(self):
super().shutdown()
self.intercom.shutdown()
self.compare_service.shutdown()
self.unpacking_service.shutdown()
self.tagging_service.shutdown()
self.analysis_service.shutdown()
if not self.args.testing:
complete_shutdown()
def _exception_occurred(self):
return any((self.unpacking_service.check_exceptions(),
self.compare_service.check_exceptions(),
self.analysis_service.check_exceptions()))
signal.signal(signal.SIGINT, lambda *_: None)
os.setpgid(os.getpid(), os.getpid()) # reset pgid to self so that "complete_shutdown" doesn't run amok
else:
signal.signal(signal.SIGINT, shutdown)
args, config = program_setup(PROGRAM_NAME, PROGRAM_DESCRIPTION)
analysis_service = AnalysisScheduler(config=config)
tagging_service = TaggingDaemon(analysis_scheduler=analysis_service)
unpacking_service = UnpackingScheduler(config=config, post_unpack=analysis_service.start_analysis_of_object, analysis_workload=analysis_service.get_scheduled_workload)
compare_service = CompareScheduler(config=config)
intercom = InterComBackEndBinding(config=config, analysis_service=analysis_service, compare_service=compare_service, unpacking_service=unpacking_service)
work_load_stat = WorkLoadStatistic(config=config)
run = True
while run:
work_load_stat.update(unpacking_workload=unpacking_service.get_scheduled_workload(), analysis_workload=analysis_service.get_scheduled_workload())
if any((unpacking_service.check_exceptions(), compare_service.check_exceptions(), analysis_service.check_exceptions())):
break
sleep(5)
if args.testing:
break
logging.info('shutdown components')
work_load_stat.shutdown()
intercom.shutdown()
compare_service.shutdown()
unpacking_service.shutdown()
tagging_service.shutdown()
analysis_service.shutdown()
if not args.testing:
complete_shutdown()
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(
config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'files_in_common': {
'all': [],
'collapse': False
}
},
'Software': {
'Compare Skipped': {
'all':
'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(config=self._config, post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_object(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value > 7:
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(file_path='{}/container/test.7z'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=10)
compare_id = unify_string_list(';'.join([fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertFalse(isinstance(result, str), 'compare result should exist')
self.assertEqual(result['plugins']['Software'], self._expected_result()['Software'])
self.assertCountEqual(result['plugins']['File_Coverage']['exclusive_files'], self._expected_result()['File_Coverage']['exclusive_files'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'exclusive_files': {
'418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787': [],
'd38970f8c5153d1041810d0908292bc8df21e7fd88aab211a8fb96c54afe6b01_319': [],
'collapse': False
},
'files_in_common': {
'all': [
'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28',
'289b5a050a83837f192d7129e4c4e02570b94b4924e50159fad5ed1067cfbfeb_20',
'd558c9339cb967341d701e3184f863d3928973fccdc1d96042583730b5c7b76a_62'
],
'collapse': False
},
'similar_files': {}
},
'Software': {
'Compare Skipped': {
'all': 'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}