def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
def setUpClass(cls):
cls._config = get_config_for_testing(TMP_DIR)
cls._config.set('data_storage', 'report_threshold', '32')
cls._config.set('data_storage', 'sanitize_database', 'tmp_sanitize')
cls.mongo_server = MongoMgr(config=cls._config)
class TestSummary(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
def tearDown(self):
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
self.db_interface_backend.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
def create_and_add_test_fimrware_and_file_object(self):
self.test_fw = create_test_firmware()
self.test_fo = create_test_file_object()
self.test_fw.add_included_file(self.test_fo)
self.db_interface_backend.add_object(self.test_fw)
self.db_interface_backend.add_object(self.test_fo)
def test_get_set_of_all_included_files(self):
self.create_and_add_test_fimrware_and_file_object()
result_set_fo = self.db_interface.get_set_of_all_included_files(
self.test_fo)
self.assertIsInstance(result_set_fo, set, 'result is not a set')
self.assertEqual(len(result_set_fo), 1, 'number of files not correct')
self.assertIn(self.test_fo.uid, result_set_fo,
'object not in its own result set')
result_set_fw = self.db_interface.get_set_of_all_included_files(
self.test_fw)
self.assertEqual(len(result_set_fw), 2, 'number of files not correct')
self.assertIn(self.test_fo.uid, result_set_fw,
'test file not in result set firmware')
self.assertIn(self.test_fw.uid, result_set_fw,
'fw not in result set firmware')
def test_get_uids_of_all_included_files(self):
def add_test_file_to_db_with_parent_uids(uid, parent_uids: Set[str]):
test_fo = create_test_file_object()
test_fo.parent_firmware_uids = parent_uids
test_fo.uid = uid
self.db_interface_backend.add_object(test_fo)
add_test_file_to_db_with_parent_uids('uid1', {'foo'})
add_test_file_to_db_with_parent_uids('uid2', {'foo', 'bar'})
add_test_file_to_db_with_parent_uids('uid3', {'bar'})
result = self.db_interface.get_uids_of_all_included_files('foo')
assert result == {'uid1', 'uid2'}
assert self.db_interface.get_uids_of_all_included_files(
'uid not in db') == set()
def test_get_summary(self):
self.create_and_add_test_fimrware_and_file_object()
result_sum = self.db_interface.get_summary(self.test_fw, 'dummy')
self.assertIsInstance(result_sum, dict, 'summary is not a dict')
self.assertIn('sum a', result_sum, 'summary entry of parent missing')
self.assertIn(self.test_fw.uid, result_sum['sum a'],
'origin (parent) missing in parent summary entry')
self.assertIn(self.test_fo.uid, result_sum['sum a'],
'origin (child) missing in parent summary entry')
self.assertNotIn(self.test_fo.uid, result_sum['fw exclusive sum a'],
'child as origin but should not be')
self.assertIn('file exclusive sum b', result_sum,
'file exclusive summary missing')
self.assertIn(self.test_fo.uid, result_sum['file exclusive sum b'],
'origin of file exclusive missing')
self.assertNotIn(self.test_fw.uid, result_sum['file exclusive sum b'],
'parent as origin but should not be')
def test_collect_summary(self):
self.create_and_add_test_fimrware_and_file_object()
fo_list = [self.test_fo.uid]
result_sum = self.db_interface._collect_summary(fo_list, 'dummy')
assert all(
item in result_sum
for item in self.test_fo.processed_analysis['dummy']['summary'])
assert all(value == [self.test_fo.uid]
for value in result_sum.values())
def test_get_summary_of_one_error_handling(self):
result_sum = self.db_interface._get_summary_of_one(None, 'foo')
self.assertEqual(result_sum, {},
'None object should result in empty dict')
self.create_and_add_test_fimrware_and_file_object()
result_sum = self.db_interface._get_summary_of_one(
self.test_fw, 'none_existing_analysis')
self.assertEqual(result_sum, {},
'analysis not existend should lead to empty dict')
def test_update_summary(self):
orig = {'a': ['a']}
update = {'a': ['aa'], 'b': ['aa']}
result = self.db_interface._update_summary(orig, update)
self.assertIn('a', result)
self.assertIn('b', result)
self.assertIn('a', result['a'])
self.assertIn('aa', result['a'])
self.assertIn('aa', result['b'])
def setUpClass(cls):
cls._set_config()
cls.mongo_server = MongoMgr(config=cls.config)
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self._mongo_server = MongoMgr(config=self._config, auth=False)
self._analysis_scheduler = AnalysisScheduler(config=self._config)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(config=self._config)
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/container/test.7z'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
sleep(10)
self._unpack_scheduler.add_task(test_fw_2)
sleep(10)
compare_id = unify_string_list(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
sleep(10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertFalse(isinstance(result, str),
'compare result should exist')
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['exclusive_files'],
self._expected_result()['File_Coverage']['exclusive_files'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'exclusive_files': {
'418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787':
[],
'd38970f8c5153d1041810d0908292bc8df21e7fd88aab211a8fb96c54afe6b01_319':
[],
'collapse':
False
},
'files_in_common': {
'all': [
'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28',
'289b5a050a83837f192d7129e4c4e02570b94b4924e50159fad5ed1067cfbfeb_20',
'd558c9339cb967341d701e3184f863d3928973fccdc1d96042583730b5c7b76a_62'
],
'collapse':
False
},
'similar_files': {}
},
'Software': {
'Compare Skipped': {
'all':
'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}
class TestFileAddition(TestCase): # pylint: disable=too-many-instance-attributes
@mock.patch('unpacker.unpack.FSOrganizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(
config=self._config,
post_unpack=self._analysis_scheduler.start_analysis_of_object)
self._compare_scheduler = CompareScheduler(
config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'files_in_common': {
'all': [],
'collapse': False
}
},
'Software': {
'Compare Skipped': {
'all': 'Required analysis not present: software_components'
}
}
}
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self.updater = StatisticUpdater(config=self.config)
self.frontend_db_interface = StatisticDbViewer(config=self.config)
def shutdown(*_):
global run
logging.info('shutting down {}...'.format(PROGRAM_NAME))
run = False
if __name__ == '__main__':
if was_started_by_start_fact():
signal.signal(signal.SIGUSR1, shutdown)
signal.signal(signal.SIGINT, lambda *_: None)
else:
signal.signal(signal.SIGINT, shutdown)
args, config = program_setup(PROGRAM_NAME, PROGRAM_DESCRIPTION)
mongo_server = MongoMgr(config=config)
work_load_stat = WorkLoadStatistic(config=config, component='database')
run = True
while run:
work_load_stat.update()
sleep(5)
if args.testing:
break
work_load_stat.shutdown()
mongo_server.shutdown()
sys.exit()
def setup(self):
self.tmp_dir = TemporaryDirectory()
self.config = get_config_for_testing(temp_dir=self.tmp_dir)
self.mongo_server = MongoMgr(config=self.config)
self._init_test_data()
self.binary_service = BinaryService(config=self.config)
class TestStorageDbInterfaceFrontend(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_frontend_interface = FrontEndDbInterface(config=self._config)
self.db_backend_interface = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
def tearDown(self):
self.db_frontend_interface.shutdown()
self.db_backend_interface.client.drop_database(self._config.get('data_storage', 'main_database'))
self.db_backend_interface.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def test_get_meta_list(self):
self.db_backend_interface.add_firmware(self.test_firmware)
list_of_firmwares = self.db_frontend_interface.get_meta_list()
test_output = list_of_firmwares.pop()
self.assertEqual(test_output[1], 'test_vendor test_router - 0.1 (Router)', 'Firmware not successfully received')
self.assertIsInstance(test_output[2], dict, 'tag field is not a dict')
def test_get_hid_firmware(self):
self.db_backend_interface.add_firmware(self.test_firmware)
result = self.db_frontend_interface.get_hid(self.test_firmware.get_uid())
self.assertEqual(result, 'test_vendor test_router - 0.1 (Router)', 'fw hid not correct')
def test_get_hid_fo(self):
test_fo = create_test_file_object(bin_path='get_files_test/testfile2')
test_fo.virtual_file_path = {'a': ['|a|/test_file'], 'b': ['|b|/get_files_test/testfile2']}
self.db_backend_interface.add_file_object(test_fo)
result = self.db_frontend_interface.get_hid(test_fo.get_uid(), root_uid='b')
self.assertEqual(result, '/get_files_test/testfile2', 'fo hid not correct')
result = self.db_frontend_interface.get_hid(test_fo.get_uid())
self.assertIsInstance(result, str, 'result is not a string')
self.assertEqual(result[0], '/', 'first character not correct if no root_uid set')
result = self.db_frontend_interface.get_hid(test_fo.get_uid(), root_uid='c')
self.assertEqual(result[0], '/', 'first character not correct if invalid root_uid set')
def test_get_file_name(self):
self.db_backend_interface.add_firmware(self.test_firmware)
result = self.db_frontend_interface.get_file_name(self.test_firmware.get_uid())
self.assertEqual(result, 'test.zip', 'name not correct')
def test_get_hid_invalid_uid(self):
result = self.db_frontend_interface.get_hid('foo')
self.assertEqual(result, '', 'invalid uid should result in empty string')
def test_get_firmware_attribute_list(self):
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertEqual(self.db_frontend_interface.get_device_class_list(), ['Router'])
self.assertEqual(self.db_frontend_interface.get_vendor_list(), ['test_vendor'])
self.assertEqual(self.db_frontend_interface.get_firmware_attribute_list('device_name', {'vendor': 'test_vendor', 'device_class': 'Router'}), ['test_router'])
self.assertEqual(self.db_frontend_interface.get_firmware_attribute_list('version'), ['0.1'])
self.assertEqual(self.db_frontend_interface.get_device_name_dict(), {'Router': {'test_vendor': ['test_router']}})
def test_get_data_for_nice_list(self):
uid_list = [self.test_firmware.get_uid()]
self.db_backend_interface.add_firmware(self.test_firmware)
nice_list_data = self.db_frontend_interface.get_data_for_nice_list(uid_list, uid_list[0])
self.assertEquals(sorted(['size', 'virtual_file_paths', 'uid', 'mime-type', 'files_included']), sorted(nice_list_data[0].keys()))
self.assertEqual(nice_list_data[0]['uid'], self.test_firmware.get_uid())
def test_generic_search(self):
self.db_backend_interface.add_firmware(self.test_firmware)
# str input
result = self.db_frontend_interface.generic_search('{"file_name": "test.zip"}')
self.assertEqual(result, [self.test_firmware.get_uid()], 'Firmware not successfully received')
# dict input
result = self.db_frontend_interface.generic_search({'file_name': 'test.zip'})
self.assertEqual(result, [self.test_firmware.get_uid()], 'Firmware not successfully received')
def test_all_uids_found_in_database(self):
self.db_backend_interface.client.drop_database(self._config.get('data_storage', 'main_database'))
uid_list = [self.test_firmware.get_uid()]
self.assertFalse(self.db_frontend_interface.all_uids_found_in_database(uid_list))
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertTrue(self.db_frontend_interface.all_uids_found_in_database([self.test_firmware.get_uid()]))
def test_get_number_of_firmwares_in_db(self):
self.assertEqual(self.db_frontend_interface.get_number_of_firmwares_in_db(), 0)
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertEqual(self.db_frontend_interface.get_number_of_firmwares_in_db(), 1)
def test_get_x_last_added_firmwares(self):
self.assertEqual(self.db_frontend_interface.get_last_added_firmwares(), [], 'empty db should result in empty list')
test_fw_one = create_test_firmware(device_name='fw_one')
self.db_backend_interface.add_firmware(test_fw_one)
test_fw_two = create_test_firmware(device_name='fw_two', bin_path='container/test.7z')
self.db_backend_interface.add_firmware(test_fw_two)
test_fw_three = create_test_firmware(device_name='fw_three', bin_path='container/test.cab')
self.db_backend_interface.add_firmware(test_fw_three)
result = self.db_frontend_interface.get_last_added_firmwares(limit_x=2)
self.assertEqual(len(result), 2, 'Number of results should be 2')
self.assertEqual(result[0]['device_name'], 'fw_three', 'last firmware is not first entry')
self.assertEqual(result[1]['device_name'], 'fw_two', 'second last firmware is not the second entry')
def test_generate_file_tree_node(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
child_fo.processed_analysis['file_type'] = {'mime': 'sometype'}
uid = parent_fw.get_uid()
child_fo.virtual_file_path = {uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]}
parent_fw.files_included = {child_fo.get_uid()}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
for node in self.db_frontend_interface.generate_file_tree_node(uid, uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, parent_fw.file_name)
self.assertTrue(node.has_children)
for node in self.db_frontend_interface.generate_file_tree_node(child_fo.get_uid(), uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, 'folder')
self.assertTrue(node.has_children)
virtual_grand_child = node.get_list_of_child_nodes()[0]
self.assertEqual(virtual_grand_child.type, 'sometype')
self.assertFalse(virtual_grand_child.has_children)
self.assertEqual(virtual_grand_child.name, child_fo.file_name)
def test_get_number_of_total_matches(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
uid = parent_fw.get_uid()
child_fo.virtual_file_path = {uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(uid, child_fo.get_uid())
self.assertEqual(self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=False), 2)
self.assertEqual(self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True), 1)
def test_get_other_versions_of_firmware(self):
parent_fw1 = create_test_firmware(version='1')
self.db_backend_interface.add_object(parent_fw1)
parent_fw2 = create_test_firmware(version='2', bin_path='container/test.7z')
self.db_backend_interface.add_object(parent_fw2)
parent_fw3 = create_test_firmware(version='3', bin_path='container/test.cab')
self.db_backend_interface.add_object(parent_fw3)
other_versions = self.db_frontend_interface.get_other_versions_of_firmware(parent_fw1)
self.assertEqual(len(other_versions), 2, 'wrong number of other versions')
self.assertIn({'_id': parent_fw2.get_uid(), 'version': '2'}, other_versions)
self.assertIn({'_id': parent_fw3.get_uid(), 'version': '3'}, other_versions)
other_versions = self.db_frontend_interface.get_other_versions_of_firmware(parent_fw2)
self.assertIn({'_id': parent_fw3.get_uid(), 'version': '3'}, other_versions)
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_frontend_interface = FrontEndDbInterface(config=self._config)
self.db_backend_interface = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
def setUpClass(cls):
cls.tmp_dir = TemporaryDirectory(prefix='fact_test_')
cls.config = get_config_for_testing(temp_dir=cls.tmp_dir)
cls.config.set('ExpertSettings', 'communication_timeout', '1')
cls.mongo_server = MongoMgr(config=cls.config)
class TestMongoInterface(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self._config.set('data_storage', 'report_threshold', '32')
self._config.set('data_storage', 'sanitize_database', 'tmp_sanitize')
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def tearDown(self):
self.db_interface_backend.client.drop_database(self._config.get('data_storage', 'main_database'))
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(self._config.get('data_storage', 'sanitize_database'))
self.db_interface.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
gc.collect()
def _get_all_firmware_uids(self):
uid_list = []
tmp = self.db_interface.firmwares.find()
for item in tmp:
uid_list.append(item['_id'])
return uid_list
def test_existence_quick_check(self):
self.assertFalse(self.db_interface.existence_quick_check('none_existing'), 'none existing firmware found')
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertTrue(self.db_interface.existence_quick_check(self.test_firmware.get_uid()), 'existing firmware not found')
self.db_interface_backend.add_file_object(self.test_fo)
self.assertTrue(self.db_interface.existence_quick_check(self.test_fo.get_uid()), 'existing file not found')
def test_get_firmware(self):
self.db_interface_backend.add_firmware(self.test_firmware)
fobject = self.db_interface.get_firmware(self.test_firmware.get_uid())
self.assertEqual(fobject.vendor, 'test_vendor')
self.assertEqual(fobject.device_name, 'test_router')
def test_get_object(self):
fo = self.db_interface.get_object(self.test_firmware.get_uid())
self.assertIsNone(fo, 'found something but there is nothing in the database')
self.db_interface_backend.add_firmware(self.test_firmware)
fo = self.db_interface.get_object(self.test_firmware.get_uid())
self.assertIsInstance(fo, Firmware, 'firmware has wrong type')
self.assertEqual(fo.device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo = self.db_interface.get_object(test_file.get_uid())
self.assertIsInstance(fo, FileObject, 'file object has wrong type')
def test_get_complete_object_including_all_summaries(self):
self.db_interface_backend.report_threshold = 1024
test_file = create_test_file_object()
self.test_firmware.add_included_file(test_file)
self.db_interface_backend.add_firmware(self.test_firmware)
self.db_interface_backend.add_file_object(test_file)
tmp = self.db_interface.get_complete_object_including_all_summaries(self.test_firmware.get_uid())
self.assertIsInstance(tmp, Firmware, 'wrong type')
self.assertIn('summary', tmp.processed_analysis['dummy'].keys(), 'summary not found in processed analysis')
self.assertIn('sum a', tmp.processed_analysis['dummy']['summary'], 'summary of original file not included')
self.assertIn('file exclusive sum b', tmp.processed_analysis['dummy']['summary'], 'summary of included file not found')
def test_sanitize_analysis(self):
short_dict = {'stub_plugin': {'result': 0}}
long_dict = {'stub_plugin': {'result': 10000000000, 'misc': 'Bananarama', 'summary': []}}
self.test_firmware.processed_analysis = short_dict
sanitized_dict = self.db_interface.sanitize_analysis(self.test_firmware.processed_analysis, self.test_firmware.get_uid())
self.assertIn('file_system_flag', sanitized_dict['stub_plugin'].keys())
self.assertFalse(sanitized_dict['stub_plugin']['file_system_flag'])
self.assertEqual(self.db_interface.sanitize_fs.list(), [], 'file stored in db but should not')
self.test_firmware.processed_analysis = long_dict
sanitized_dict = self.db_interface.sanitize_analysis(self.test_firmware.processed_analysis, self.test_firmware.get_uid())
self.assertIn('stub_plugin_result_{}'.format(self.test_firmware.get_uid()), self.db_interface.sanitize_fs.list(), 'sanitized file not stored')
self.assertNotIn('summary_result_{}'.format(self.test_firmware.get_uid()), self.db_interface.sanitize_fs.list(), 'summary is erroneously stored')
self.assertIn('file_system_flag', sanitized_dict['stub_plugin'].keys())
self.assertTrue(sanitized_dict['stub_plugin']['file_system_flag'])
self.assertEqual(type(sanitized_dict['stub_plugin']['summary']), list)
def test_retrieve_analysis(self):
self.db_interface.sanitize_fs.put(pickle.dumps('This is a test!'), filename='test_file_path')
sanitized_dict = {'stub_plugin': {'result': 'test_file_path', 'file_system_flag': True}}
sanitized_dict['inbound_result'] = {'result': 'inbound result', 'file_system_flag': False}
retrieved_dict = self.db_interface.retrieve_analysis(sanitized_dict)
self.assertNotIn('file_system_flag', retrieved_dict['stub_plugin'].keys())
self.assertIn('result', retrieved_dict['stub_plugin'].keys())
self.assertEqual(retrieved_dict['stub_plugin']['result'], 'This is a test!')
self.assertNotIn('file_system_flag', retrieved_dict['inbound_result'].keys())
self.assertEqual(retrieved_dict['inbound_result']['result'], 'inbound result')
def test_retrieve_analysis_filter(self):
self.db_interface.sanitize_fs.put(pickle.dumps('This is a test!'), filename='test_file_path')
sanitized_dict = {'selected_plugin': {'result': 'test_file_path', 'file_system_flag': True}}
sanitized_dict['other_plugin'] = {'result': 'test_file_path', 'file_system_flag': True}
retrieved_dict = self.db_interface.retrieve_analysis(sanitized_dict, analysis_filter=['selected_plugin'])
self.assertEqual(retrieved_dict['selected_plugin']['result'], 'This is a test!')
self.assertIn('file_system_flag', retrieved_dict['other_plugin'])
def test_get_objects_by_uid_list(self):
self.db_interface_backend.add_firmware(self.test_firmware)
fo_list = self.db_interface.get_objects_by_uid_list([self.test_firmware.get_uid()])
self.assertIsInstance(fo_list[0], Firmware, 'firmware has wrong type')
self.assertEqual(fo_list[0].device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo_list = self.db_interface.get_objects_by_uid_list([test_file.get_uid()])
self.assertIsInstance(fo_list[0], FileObject, 'file object has wrong type')
def test_sanitize_extract_and_retrieve_binary(self):
test_data = {'dummy': {'test_key': 'test_value'}}
test_data['dummy'] = self.db_interface._extract_binaries(test_data, 'dummy', 'uid')
self.assertEqual(self.db_interface.sanitize_fs.list(), ['dummy_test_key_uid'], 'file not written')
self.assertEqual(test_data['dummy']['test_key'], 'dummy_test_key_uid', 'new file path not set')
test_data['dummy'] = self.db_interface._retrieve_binaries(test_data, 'dummy')
self.assertEqual(test_data['dummy']['test_key'], 'test_value', 'value not recoverd')
def test_get_firmware_number(self):
result = self.db_interface.get_firmware_number(query={})
self.assertEqual(result, 0)
self.db_interface_backend.add_firmware(self.test_firmware)
result = self.db_interface.get_firmware_number(query={})
self.assertEqual(result, 1)
result = self.db_interface.get_firmware_number(query={'_id': self.test_firmware.uid})
self.assertEqual(result, 1)
test_fw_2 = create_test_firmware(bin_path='container/test.7z')
self.db_interface_backend.add_firmware(test_fw_2)
result = self.db_interface.get_firmware_number(query='{}')
self.assertEqual(result, 2)
result = self.db_interface.get_firmware_number(query={'_id': self.test_firmware.uid})
self.assertEqual(result, 1)
def test_get_file_object_number(self):
result = self.db_interface.get_file_object_number(query={})
self.assertEqual(result, 0)
self.db_interface_backend.add_file_object(self.test_fo)
result = self.db_interface.get_file_object_number(query={}, zero_on_empty_query=False)
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(query={'_id': self.test_fo.uid})
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(query=json.dumps({'_id': self.test_fo.uid}))
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(query={}, zero_on_empty_query=True)
self.assertEqual(result, 0)
result = self.db_interface.get_file_object_number(query='{}', zero_on_empty_query=True)
self.assertEqual(result, 0)
test_fo_2 = create_test_file_object(bin_path='get_files_test/testfile2')
self.db_interface_backend.add_file_object(test_fo_2)
result = self.db_interface.get_file_object_number(query={}, zero_on_empty_query=False)
self.assertEqual(result, 2)
result = self.db_interface.get_file_object_number(query={'_id': self.test_fo.uid})
self.assertEqual(result, 1)
class TestSummary(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
def tearDown(self):
self.db_interface.client.drop_database(self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
self.db_interface_backend.shutdown()
self.mongo_server.shutdown()
TMP_DIR.cleanup()
def create_and_add_test_fimrware_and_file_object(self):
self.test_fw = create_test_firmware()
self.test_fo = create_test_file_object()
self.test_fw.add_included_file(self.test_fo)
self.db_interface_backend.add_object(self.test_fw)
self.db_interface_backend.add_object(self.test_fo)
def get_set_of_all_included_files(self):
self.create_and_add_test_fimrware_and_file_object()
result_set_fo = self.db_interface.get_set_of_all_included_files(self.test_fo)
self.assertIsInstance(result_set_fo, set, 'result is not a set')
self.assertEqual(len(result_set_fo), 1, 'number of files not correct')
self.assertIn(self.test_fo.get_uid(), result_set_fo, 'object not in its own result set')
result_set_fw = self.db_interface.get_set_of_all_included_files(self.test_fw)
self.assertEqual(len(result_set_fw), 2, 'number of files not correct')
self.assertIn(self.test_fo.get_uid(), result_set_fw, 'test file not in result set firmware')
self.assertIn(self.test_fw.get_uid(), result_set_fw, 'fw not in result set firmware')
def test_get_summary(self):
self.create_and_add_test_fimrware_and_file_object()
result_sum = self.db_interface.get_summary(self.test_fw, 'dummy')
self.assertIsInstance(result_sum, dict, 'summary is not a dict')
self.assertIn('sum a', result_sum, 'summary entry of parent missing')
self.assertIn(self.test_fw.get_uid(), result_sum['sum a'], 'origin (parent) missing in parent summary entry')
self.assertIn(self.test_fo.get_uid(), result_sum['sum a'], 'origin (child) missing in parent summary entry')
self.assertNotIn(self.test_fo.get_uid(), result_sum['fw exclusive sum a'], 'child as origin but should not be')
self.assertIn('file exclusive sum b', result_sum, 'file exclusive summary missing')
self.assertIn(self.test_fo.get_uid(), result_sum['file exclusive sum b'], 'origin of file exclusive missing')
self.assertNotIn(self.test_fw.get_uid(), result_sum['file exclusive sum b'], 'parent as origin but should not be')
def test_get_summary_of_one_error_handling(self):
result_sum = self.db_interface._get_summary_of_one(None, 'foo')
self.assertEqual(result_sum, {}, 'None object should result in empty dict')
self.create_and_add_test_fimrware_and_file_object()
result_sum = self.db_interface._get_summary_of_one(self.test_fw, 'none_existing_analysis')
self.assertEqual(result_sum, {}, 'analysis not existend should lead to empty dict')
def test_update_summary(self):
orig = {'a': ['a']}
update = {'a': ['aa'], 'b': ['aa']}
result = self.db_interface._update_summary(orig, update)
self.assertIn('a', result)
self.assertIn('b', result)
self.assertIn('a', result['a'])
self.assertIn('aa', result['a'])
self.assertIn('aa', result['b'])
class TestCompare(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
def tearDown(self):
self.db_interface_compare.shutdown()
self.db_interface_admin.shutdown()
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
self.mongo_server.shutdown()
gc.collect()
def _create_compare_dict(self):
comp_dict = {
'general': {
'hid': {
self.fw_one.get_uid(): 'foo',
self.fw_two.get_uid(): 'bar'
}
},
'plugins': {}
}
comp_dict['general']['virtual_file_path'] = {
self.fw_one.get_uid(): 'dev_one_name',
self.fw_two.get_uid(): 'dev_two_name'
}
return comp_dict
def test_add_and_get_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
retrieved = self.db_interface_compare.get_compare_result(
'{};{}'.format(self.fw_one.get_uid(), self.fw_two.get_uid()))
self.assertEqual(
retrieved['general']['virtual_file_path'][self.fw_one.get_uid()],
'dev_one_name', 'content of retrieval not correct')
def test_get_not_existing_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
result = self.db_interface_compare.get_compare_result('{};{}'.format(
self.fw_one.get_uid(), self.fw_two.get_uid()))
self.assertIsNone(result, 'result not none')
def test_calculate_compare_result_id(self):
comp_id = self.db_interface_compare._calculate_compare_result_id(
self.compare_dict)
self.assertEqual(
comp_id, '{};{}'.format(self.fw_one.get_uid(),
self.fw_two.get_uid()))
def test_object_existence_quick_check(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.assertIsNone(
self.db_interface_compare.object_existence_quick_check(
self.fw_one.get_uid()), 'existing_object not found')
self.assertEqual(
self.db_interface_compare.object_existence_quick_check(
'{};none_existing_object'.format(self.fw_one.get_uid())),
'none_existing_object not found in database',
'error message not correct')
def test_get_compare_result_of_none_existing_uid(self):
self.db_interface_backend.add_firmware(self.fw_one)
result = self.db_interface_compare.get_compare_result(
'{};none_existing_uid'.format(self.fw_one.get_uid()))
self.assertEqual(result, 'none_existing_uid not found in database',
'no result not found error')
def test_get_latest_comparisons(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
before = time()
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
for id, hids, submission_date in result:
self.assertIn(self.fw_one.get_uid(), hids)
self.assertIn(self.fw_two.get_uid(), hids)
self.assertIn(self.fw_one.get_uid(), id)
self.assertIn(self.fw_two.get_uid(), id)
self.assertTrue(before <= submission_date <= time())
def test_get_latest_comparisons_removed_firmware(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertNotEqual(result, [], 'A compare result should be available')
self.db_interface_admin.delete_firmware(self.fw_two.uid)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertEqual(result, [], 'No compare result should be available')
def test_get_total_number_of_results(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
number = self.db_interface_compare.get_total_number_of_results()
self.assertEqual(number, 1, 'no compare result found in database')
def setUpClass(cls):
cls._config = get_config_for_testing(TMP_DIR)
cls.mongo_server = MongoMgr(config=cls._config)
def setup_class(cls):
cls._config = get_config_for_testing()
cls.mongo_server = MongoMgr(config=cls._config)
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self._init_test_data()
self.binary_service = BinaryService(config=self.config)
class TestTagPropagation(unittest.TestCase):
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.analysis_finished_event = Event()
self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048'
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
pre_analysis=self.backend_interface.add_object,
post_analysis=self.count_analysis_finished_event)
self._tagging_scheduler = TaggingDaemon(
analysis_scheduler=self._analysis_scheduler)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
if fw_object.uid == self.uid_of_key_file and 'crypto_material' in fw_object.processed_analysis:
sleep(1)
self.analysis_finished_event.set()
def _wait_for_empty_tag_queue(self):
while not self._analysis_scheduler.tag_queue.empty():
sleep(0.1)
def tearDown(self):
self._unpack_scheduler.shutdown()
self._tagging_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_run_analysis_with_tag(self):
test_fw = Firmware(
file_path='{}/container/with_key.7z'.format(get_test_data_dir()))
test_fw.release_date = '2017-01-01'
test_fw.scheduled_analysis = ['crypto_material']
self._unpack_scheduler.add_task(test_fw)
assert self.analysis_finished_event.wait(timeout=20)
processed_fo = self.backend_interface.get_object(
self.uid_of_key_file, analysis_filter=['crypto_material'])
assert processed_fo.processed_analysis['crypto_material'][
'tags'], 'no tags set in analysis'
self._wait_for_empty_tag_queue()
processed_fw = self.backend_interface.get_object(
test_fw.uid, analysis_filter=['crypto_material'])
assert processed_fw.analysis_tags, 'tags not propagated properly'
assert processed_fw.analysis_tags['crypto_material'][
'private_key_inside']
class TestStatistic(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self.updater = StatisticUpdater(config=self.config)
self.frontend_db_interface = StatisticDbViewer(config=self.config)
def tearDown(self):
self.updater.shutdown()
self.frontend_db_interface.shutdown()
clean_test_database(self.config, get_database_names(self.config))
self.mongo_server.shutdown()
gc.collect()
def test_update_and_get_statistic(self):
self.updater.db.update_statistic('test', {'test1': 1})
result = self.frontend_db_interface.get_statistic('test')
self.assertEqual(result['test1'], 1, 'result not correct')
self.updater.db.update_statistic('test', {'test1': 2})
result = self.frontend_db_interface.get_statistic('test')
self.assertEqual(result['test1'], 2, 'result not correct')
def test_get_general_stats(self):
result = self.updater.get_general_stats()
self.assertEqual(result['number_of_firmwares'], 0,
'number of firmwares not correct')
self.assertEqual(result['number_of_unique_files'], 0,
'number of files not correct')
self.updater.db.firmwares.insert_one({'test': 1})
self.updater.db.file_objects.insert_one({'test': 1})
result = self.updater.get_general_stats()
self.assertEqual(result['number_of_firmwares'], 1,
'number of firmwares not correct')
self.assertEqual(result['number_of_unique_files'], 1,
'number of files not correct')
def test_convert_dict_list_to_list(self):
test_list = [{
'count': 1,
'_id': 'A'
}, {
'count': 2,
'_id': 'B'
}, {
'count': 3,
'_id': None
}]
result = self.updater._convert_dict_list_to_list(test_list)
self.assertIsInstance(result, list, 'result is not a list')
self.assertIn(['A', 1], result)
self.assertIn(['B', 2], result)
self.assertIn(['not available', 3], result)
self.assertEqual(len(result), 3, 'too many keys in the result')
def test_filter_sanitized_entries(self):
test_list = [
['valid', 1],
[
'sanitized_81abfc7a79c8c1ed85f6b9fc2c5d9a3edc4456c4aecb9f95b4d7a2bf9bf652da_1',
1
]
]
result = self.updater._filter_sanitzized_objects(test_list)
self.assertEqual(result, [['valid', 1]])
def test_find_most_frequent_architecture(self):
test_list = [
'MIPS, 32-bit, big endian (M)', 'MIPS (M)',
'MIPS, 32-bit, big endian (M)', 'MIPS, 32-bit, big endian (M)'
]
result = self.updater._find_most_frequent_architecture(test_list)
expected_result = 'MIPS, 32-bit, big endian (M)'
self.assertEqual(result, expected_result)
test_list = ['A', 'B', 'B', 'B', 'C', 'C']
result = self.updater._find_most_frequent_architecture(test_list)
expected_result = 'B'
self.assertEqual(result, expected_result)
def test_count_occurrences(self):
test_list = ['A', 'B', 'B', 'C', 'C', 'C']
result = set(self.updater._count_occurrences(test_list))
expected_result = {('A', 1), ('C', 3), ('B', 2)}
self.assertEqual(result, expected_result)
def test_shorten_architecture_string(self):
tests_string = 'MIPS, 64-bit, little endian (M)'
result = self.updater._shorten_architecture_string(tests_string)
self.assertEqual(result, 'MIPS, 64-bit')
tests_string = 'MIPS (M)'
result = self.updater._shorten_architecture_string(tests_string)
self.assertEqual(result, 'MIPS')
