def setUp(self):
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
def setUp(self):
self.admin_interface = AdminDbInterface(config=self.config)
self.db_backend_interface = BackEndDbInterface(config=self.config)
copyfile(test_firmware_original, test_firmware_copy)
self.test_firmware = create_test_firmware(
bin_path='container/test_copy.zip')
self.uid = self.test_firmware.get_uid()
self.test_firmware.virtual_file_path = {
self.uid: ['|{}|'.format(self.test_firmware.get_uid())]
}
copyfile(test_file_original, test_file_copy)
self.child_fo = create_test_file_object(test_file_copy)
self.child_fo.virtual_file_path = {
self.uid:
['|{}|/folder/{}'.format(self.uid, self.child_fo.file_name)]
}
self.test_firmware.files_included = [self.child_fo.get_uid()]
self.child_uid = self.child_fo.get_uid()
def setUp(self):
self.admin_interface = AdminDbInterface(config=self.config)
self.db_backend_interface = BackEndDbInterface(config=self.config)
copyfile(TEST_FIRMWARE_ORIGINAL, TEST_FIRMWARE_COPY)
self.test_firmware = create_test_firmware(
bin_path='container/test_copy.zip')
self.uid = self.test_firmware.uid
self.test_firmware.virtual_file_path = {
self.uid: ['|{}|'.format(self.test_firmware.uid)]
}
copyfile(TEST_FILE_ORIGINAL, TEST_FILE_COPY)
self.child_fo = create_test_file_object(TEST_FILE_COPY)
self.child_fo.virtual_file_path = {
self.uid:
['|{}|/folder/{}'.format(self.uid, self.child_fo.file_name)]
}
self.test_firmware.files_included = [self.child_fo.uid]
self.child_uid = self.child_fo.uid
class TestStorageDbInterfaceAdmin(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.config = get_config_for_testing(TMP_DIR)
cls.config.set('data_storage', 'sanitize_database', 'tmp_sanitize')
cls.config.set('data_storage', 'report_threshold', '32')
cls.mongo_server = MongoMgr(config=cls.config)
def setUp(self):
self.admin_interface = AdminDbInterface(config=self.config)
self.db_backend_interface = BackEndDbInterface(config=self.config)
copyfile(test_firmware_original, test_firmware_copy)
self.test_firmware = create_test_firmware(
bin_path='container/test_copy.zip')
self.uid = self.test_firmware.get_uid()
self.test_firmware.virtual_file_path = {
self.uid: ['|{}|'.format(self.test_firmware.get_uid())]
}
copyfile(test_file_original, test_file_copy)
self.child_fo = create_test_file_object(test_file_copy)
self.child_fo.virtual_file_path = {
self.uid:
['|{}|/folder/{}'.format(self.uid, self.child_fo.file_name)]
}
self.test_firmware.files_included = [self.child_fo.get_uid()]
self.child_uid = self.child_fo.get_uid()
def tearDown(self):
self.admin_interface.client.drop_database(
self.config.get('data_storage', 'main_database'))
self.admin_interface.client.drop_database(
self.config.get('data_storage', 'sanitize_database'))
self.admin_interface.shutdown()
self.db_backend_interface.shutdown()
gc.collect()
@classmethod
def tearDownClass(cls):
cls.mongo_server.shutdown()
for test_file in [test_file_copy, test_firmware_copy]:
if os.path.isfile(test_file):
os.remove(test_file)
TMP_DIR.cleanup()
def test_remove_object_field(self):
self.db_backend_interface.add_file_object(self.child_fo)
self.assertIn(
self.uid,
self.db_backend_interface.file_objects.find_one(
self.child_uid, {'virtual_file_path': 1})['virtual_file_path'])
self.admin_interface.remove_object_field(
self.child_uid, 'virtual_file_path.{}'.format(self.uid))
self.assertNotIn(
self.uid,
self.db_backend_interface.file_objects.find_one(
self.child_uid, {'virtual_file_path': 1})['virtual_file_path'])
def test_remove_virtual_path_entries_no_other_roots(self):
self.db_backend_interface.add_file_object(self.child_fo)
self.assertIn(
self.uid,
self.db_backend_interface.file_objects.find_one(
self.child_uid, {'virtual_file_path': 1})['virtual_file_path'])
removed_vps, deleted_files = self.admin_interface._remove_virtual_path_entries(
self.uid, self.child_fo.get_uid())
self.assertIsNone(
self.db_backend_interface.file_objects.find_one(self.child_uid))
self.assertEqual(removed_vps, 0)
self.assertEqual(deleted_files, 1)
def test_remove_virtual_path_entries_other_roots(self):
self.child_fo.virtual_file_path.update(
{'someuid': ['|someuid|/some/virtual/path']})
self.db_backend_interface.add_file_object(self.child_fo)
self.assertIn(
self.uid,
self.db_backend_interface.file_objects.find_one(
self.child_uid, {'virtual_file_path': 1})['virtual_file_path'])
removed_vps, deleted_files = self.admin_interface._remove_virtual_path_entries(
self.uid, self.child_fo.get_uid())
self.assertNotIn(
self.uid,
self.db_backend_interface.file_objects.find_one(
self.child_uid, {'virtual_file_path': 1})['virtual_file_path'])
self.assertEqual(removed_vps, 1)
self.assertEqual(deleted_files, 0)
def test_delete_swapped_analysis_entries(self):
self.test_firmware.processed_analysis = {
'test_plugin': {
'result': 10000000000,
'misc': 'delete_swap_test'
}
}
self.db_backend_interface.add_firmware(self.test_firmware)
self.admin_interface.client.drop_database(
self.config.get('data_storage', 'sanitize_database'))
self.admin_interface.sanitize_analysis(
self.test_firmware.processed_analysis, self.uid)
self.assertIn(
'test_plugin_result_{}'.format(self.test_firmware.get_uid()),
self.admin_interface.sanitize_fs.list())
self.admin_interface._delete_swapped_analysis_entries(
self.admin_interface.firmwares.find_one(self.uid))
self.assertNotIn(
'test_plugin_result_{}'.format(self.test_firmware.get_uid()),
self.admin_interface.sanitize_fs.list())
def test_delete_file_object(self):
self.db_backend_interface.add_file_object(self.child_fo)
db_entry = self.db_backend_interface.file_objects.find_one(
self.child_fo.get_uid())
self.assertIsNotNone(db_entry)
self.admin_interface._delete_file_object(db_entry)
self.assertIsNone(
self.db_backend_interface.file_objects.find_one(
self.child_fo.get_uid()), 'file not deleted from db')
delete_tasks = self._get_delete_tasks()
self.assertIn(self.child_fo.get_uid(), delete_tasks,
'file not found in delete tasks')
def test_delete_firmware(self):
self.db_backend_interface.add_firmware(self.test_firmware)
self.db_backend_interface.add_file_object(self.child_fo)
self.assertIsNotNone(
self.db_backend_interface.firmwares.find_one(self.uid))
self.assertIsNotNone(
self.db_backend_interface.file_objects.find_one(self.child_uid))
self.assertTrue(os.path.isfile(self.test_firmware.file_path))
self.assertTrue(os.path.isfile(self.child_fo.file_path))
removed_vps, deleted_files = self.admin_interface.delete_firmware(
self.uid)
self.assertIsNone(
self.db_backend_interface.firmwares.find_one(self.uid),
'firmware not deleted from db')
self.assertIsNone(
self.db_backend_interface.file_objects.find_one(self.child_uid),
'child not deleted from db')
self.assertEqual(removed_vps, 0)
self.assertEqual(deleted_files, 2,
'number of removed files not correct')
# check if file delete tasks were created
delete_tasks = self._get_delete_tasks()
self.assertIn(self.test_firmware.get_uid(), delete_tasks,
'fw delete task not found')
self.assertIn(self.child_fo.get_uid(), delete_tasks,
'child delete task not found')
self.assertEqual(len(delete_tasks), 2,
'number of delete tasks not correct')
def _get_delete_tasks(self):
intercom = InterComListener(config=self.config)
intercom.CONNECTION_TYPE = 'file_delete_task'
delete_tasks = []
while True:
tmp = intercom.get_next_task()
if tmp is None:
break
else:
delete_tasks.append(tmp['_id'])
intercom.shutdown()
return delete_tasks
class TestCompare(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._config = get_config_for_testing()
cls.mongo_server = MongoMgr(config=cls._config)
def setUp(self):
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
def tearDown(self):
self.db_interface_compare.shutdown()
self.db_interface_admin.shutdown()
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
gc.collect()
@classmethod
def tearDownClass(cls):
cls.mongo_server.shutdown()
def _create_compare_dict(self):
comp_dict = {
'general': {
'hid': {
self.fw_one.uid: 'foo',
self.fw_two.uid: 'bar'
}
},
'plugins': {}
}
comp_dict['general']['virtual_file_path'] = {
self.fw_one.uid: 'dev_one_name',
self.fw_two.uid: 'dev_two_name'
}
return comp_dict
def test_add_and_get_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
retrieved = self.db_interface_compare.get_compare_result(
'{};{}'.format(self.fw_one.uid, self.fw_two.uid))
self.assertEqual(
retrieved['general']['virtual_file_path'][self.fw_one.uid],
'dev_one_name', 'content of retrieval not correct')
def test_get_not_existing_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
result = self.db_interface_compare.get_compare_result('{};{}'.format(
self.fw_one.uid, self.fw_two.uid))
self.assertIsNone(result, 'result not none')
def test_calculate_compare_result_id(self):
comp_id = self.db_interface_compare._calculate_compare_result_id(
self.compare_dict)
self.assertEqual(comp_id, '{};{}'.format(self.fw_one.uid,
self.fw_two.uid))
def test_calculate_compare_result_id__incomplete_entries(self):
compare_dict = {
'general': {
'stat_1': {
'a': None
},
'stat_2': {
'b': None
}
}
}
comp_id = self.db_interface_compare._calculate_compare_result_id(
compare_dict)
self.assertEqual('a;b', comp_id)
def test_check_objects_exist(self):
self.db_interface_backend.add_firmware(self.fw_one)
assert self.db_interface_compare.check_objects_exist(
self.fw_one.uid) is None, 'existing_object not found'
with pytest.raises(FactCompareException):
self.db_interface_compare.check_objects_exist(
'{};none_existing_object'.format(self.fw_one.uid))
def test_get_compare_result_of_nonexistent_uid(self):
self.db_interface_backend.add_firmware(self.fw_one)
try:
self.db_interface_compare.check_objects_exist(
'{};none_existing_object'.format(self.fw_one.uid))
except FactCompareException as exception:
assert exception.get_message(
) == 'none_existing_object not found in database', 'error message not correct'
def test_get_latest_comparisons(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
before = time()
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
for id_, hids, submission_date in result:
self.assertIn(self.fw_one.uid, hids)
self.assertIn(self.fw_two.uid, hids)
self.assertIn(self.fw_one.uid, id_)
self.assertIn(self.fw_two.uid, id_)
self.assertTrue(before <= submission_date <= time())
def test_get_latest_comparisons_removed_firmware(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertNotEqual(result, [], 'A compare result should be available')
self.db_interface_admin.delete_firmware(self.fw_two.uid)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertEqual(result, [], 'No compare result should be available')
def test_get_total_number_of_results(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
number = self.db_interface_compare.get_total_number_of_results()
self.assertEqual(number, 1, 'no compare result found in database')
class TestCompare:
@classmethod
def setup_class(cls):
cls._config = get_config_for_testing()
cls.mongo_server = MongoMgr(config=cls._config)
def setup(self):
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
self.compare_id = '{};{}'.format(self.fw_one.uid, self.fw_two.uid)
def teardown(self):
self.db_interface_compare.shutdown()
self.db_interface_admin.shutdown()
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
gc.collect()
@classmethod
def teardown_class(cls):
cls.mongo_server.shutdown()
def _create_compare_dict(self):
return {
'general': {
'hid': {
self.fw_one.uid: 'foo',
self.fw_two.uid: 'bar'
},
'virtual_file_path': {
self.fw_one.uid: 'dev_one_name',
self.fw_two.uid: 'dev_two_name'
}
},
'plugins': {},
}
def test_add_and_get_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
retrieved = self.db_interface_compare.get_compare_result(
self.compare_id)
assert retrieved['general']['virtual_file_path'][self.fw_one.uid] == 'dev_one_name',\
'content of retrieval not correct'
def test_get_not_existing_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
result = self.db_interface_compare.get_compare_result(self.compare_id)
assert result is None, 'result not none'
def test_calculate_compare_result_id(self):
comp_id = self.db_interface_compare._calculate_compare_result_id(
self.compare_dict)
assert comp_id == self.compare_id
def test_calculate_compare_result_id__incomplete_entries(self):
compare_dict = {
'general': {
'stat_1': {
'a': None
},
'stat_2': {
'b': None
}
}
}
comp_id = self.db_interface_compare._calculate_compare_result_id(
compare_dict)
assert comp_id == 'a;b'
def test_check_objects_exist(self):
self.db_interface_backend.add_firmware(self.fw_one)
assert self.db_interface_compare.check_objects_exist(
self.fw_one.uid) is None, 'existing_object not found'
with pytest.raises(FactCompareException):
self.db_interface_compare.check_objects_exist(
'{};none_existing_object'.format(self.fw_one.uid))
def test_get_compare_result_of_nonexistent_uid(self):
self.db_interface_backend.add_firmware(self.fw_one)
try:
self.db_interface_compare.check_objects_exist(
'{};none_existing_object'.format(self.fw_one.uid))
except FactCompareException as exception:
assert exception.get_message(
) == 'none_existing_object not found in database', 'error message not correct'
def test_get_latest_comparisons(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
before = time()
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
for id_, hids, submission_date in result:
assert self.fw_one.uid in hids
assert self.fw_two.uid in hids
assert self.fw_one.uid in id_
assert self.fw_two.uid in id_
assert before <= submission_date <= time()
def test_get_latest_comparisons_removed_firmware(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
assert result != [], 'A compare result should be available'
self.db_interface_admin.delete_firmware(self.fw_two.uid)
result = self.db_interface_compare.page_compare_results(limit=10)
assert result == [], 'No compare result should be available'
def test_get_total_number_of_results(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
number = self.db_interface_compare.get_total_number_of_results()
assert number == 1, 'no compare result found in database'
@pytest.mark.parametrize('root_uid, expected_result', [
('the_root_uid', ['uid1', 'uid2']),
('some_other_uid', []),
(None, []),
])
def test_get_exclusive_files(self, root_uid, expected_result):
compare_dict = self._create_compare_dict()
compare_dict['plugins'] = {
'File_Coverage': {
'exclusive_files': {
'the_root_uid': ['uid1', 'uid2']
}
}
}
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(compare_dict)
exclusive_files = self.db_interface_compare.get_exclusive_files(
self.compare_id, root_uid)
assert exclusive_files == expected_result
class TestCompare(unittest.TestCase):
def setUp(self):
self._config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
def tearDown(self):
self.db_interface_compare.shutdown()
self.db_interface_admin.shutdown()
self.db_interface_backend.shutdown()
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface.shutdown()
self.mongo_server.shutdown()
gc.collect()
def _create_compare_dict(self):
comp_dict = {
'general': {
'hid': {
self.fw_one.get_uid(): 'foo',
self.fw_two.get_uid(): 'bar'
}
},
'plugins': {}
}
comp_dict['general']['virtual_file_path'] = {
self.fw_one.get_uid(): 'dev_one_name',
self.fw_two.get_uid(): 'dev_two_name'
}
return comp_dict
def test_add_and_get_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
retrieved = self.db_interface_compare.get_compare_result(
'{};{}'.format(self.fw_one.get_uid(), self.fw_two.get_uid()))
self.assertEqual(
retrieved['general']['virtual_file_path'][self.fw_one.get_uid()],
'dev_one_name', 'content of retrieval not correct')
def test_get_not_existing_compare_result(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
result = self.db_interface_compare.get_compare_result('{};{}'.format(
self.fw_one.get_uid(), self.fw_two.get_uid()))
self.assertIsNone(result, 'result not none')
def test_calculate_compare_result_id(self):
comp_id = self.db_interface_compare._calculate_compare_result_id(
self.compare_dict)
self.assertEqual(
comp_id, '{};{}'.format(self.fw_one.get_uid(),
self.fw_two.get_uid()))
def test_object_existence_quick_check(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.assertIsNone(
self.db_interface_compare.object_existence_quick_check(
self.fw_one.get_uid()), 'existing_object not found')
self.assertEqual(
self.db_interface_compare.object_existence_quick_check(
'{};none_existing_object'.format(self.fw_one.get_uid())),
'none_existing_object not found in database',
'error message not correct')
def test_get_compare_result_of_none_existing_uid(self):
self.db_interface_backend.add_firmware(self.fw_one)
result = self.db_interface_compare.get_compare_result(
'{};none_existing_uid'.format(self.fw_one.get_uid()))
self.assertEqual(result, 'none_existing_uid not found in database',
'no result not found error')
def test_get_latest_comparisons(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
before = time()
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
for id, hids, submission_date in result:
self.assertIn(self.fw_one.get_uid(), hids)
self.assertIn(self.fw_two.get_uid(), hids)
self.assertIn(self.fw_one.get_uid(), id)
self.assertIn(self.fw_two.get_uid(), id)
self.assertTrue(before <= submission_date <= time())
def test_get_latest_comparisons_removed_firmware(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertNotEqual(result, [], 'A compare result should be available')
self.db_interface_admin.delete_firmware(self.fw_two.uid)
result = self.db_interface_compare.page_compare_results(limit=10)
self.assertEqual(result, [], 'No compare result should be available')
def test_get_total_number_of_results(self):
self.db_interface_backend.add_firmware(self.fw_one)
self.db_interface_backend.add_firmware(self.fw_two)
self.db_interface_compare.add_compare_result(self.compare_dict)
number = self.db_interface_compare.get_total_number_of_results()
self.assertEqual(number, 1, 'no compare result found in database')