class TestTagPropagation(unittest.TestCase): def setUp(self): self._tmp_dir = TemporaryDirectory() self._config = initialize_config(self._tmp_dir) self.analysis_finished_event = Event() self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048' self._mongo_server = MongoMgr(config=self._config, auth=False) self.backend_interface = BackEndDbInterface(config=self._config) self._analysis_scheduler = AnalysisScheduler(config=self._config, pre_analysis=self.backend_interface.add_object, post_analysis=self.count_analysis_finished_event) self._tagging_scheduler = TaggingDaemon(analysis_scheduler=self._analysis_scheduler) self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.start_analysis_of_object) def count_analysis_finished_event(self, fw_object): self.backend_interface.add_analysis(fw_object) if fw_object.uid == self.uid_of_key_file and 'crypto_material' in fw_object.processed_analysis: sleep(1) self.analysis_finished_event.set() def _wait_for_empty_tag_queue(self): while not self._analysis_scheduler.tag_queue.empty(): sleep(0.1) def tearDown(self): self._unpack_scheduler.shutdown() self._tagging_scheduler.shutdown() self._analysis_scheduler.shutdown() clean_test_database(self._config, get_database_names(self._config)) self._mongo_server.shutdown() self._tmp_dir.cleanup() gc.collect() def test_run_analysis_with_tag(self): test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir())) test_fw.release_date = '2017-01-01' test_fw.scheduled_analysis = ['crypto_material'] self._unpack_scheduler.add_task(test_fw) assert self.analysis_finished_event.wait(timeout=20) processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material']) assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis' self._wait_for_empty_tag_queue() processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material']) assert processed_fw.analysis_tags, 'tags not propagated properly' assert processed_fw.analysis_tags['crypto_material']['private_key_inside']
class TestFileAddition(unittest.TestCase): @patch('unpacker.unpack.FS_Organizer', MockFSOrganizer) def setUp(self): self._tmp_dir = TemporaryDirectory() self._config = initialize_config(self._tmp_dir) self.elements_finished_analyzing = Value('i', 0) self.analysis_finished_event = Event() self.compare_finished_event = Event() self._mongo_server = MongoMgr(config=self._config, auth=False) self.backend_interface = BackEndDbInterface(config=self._config) self._analysis_scheduler = AnalysisScheduler( config=self._config, post_analysis=self.count_analysis_finished_event) self._unpack_scheduler = UnpackingScheduler( config=self._config, post_unpack=self._analysis_scheduler.add_task) self._compare_scheduler = CompareScheduler( config=self._config, callback=self.trigger_compare_finished_event) def count_analysis_finished_event(self, fw_object): self.backend_interface.add_analysis(fw_object) self.elements_finished_analyzing.value += 1 if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins self.analysis_finished_event.set() def trigger_compare_finished_event(self): self.compare_finished_event.set() def tearDown(self): self._compare_scheduler.shutdown() self._unpack_scheduler.shutdown() self._analysis_scheduler.shutdown() clean_test_database(self._config, get_database_names(self._config)) self._mongo_server.shutdown() self._tmp_dir.cleanup() gc.collect() def test_unpack_analyse_and_compare(self): test_fw_1 = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) test_fw_1.release_date = '2017-01-01' test_fw_2 = Firmware( file_path='{}/regression_one'.format(get_test_data_dir())) test_fw_2.release_date = '2017-01-01' self._unpack_scheduler.add_task(test_fw_1) self._unpack_scheduler.add_task(test_fw_2) self.analysis_finished_event.wait(timeout=20) compare_id = normalize_compare_id(';'.join( [fw.uid for fw in [test_fw_1, test_fw_2]])) self.assertIsNone( self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error') self.compare_finished_event.wait(timeout=10) with ConnectTo(CompareDbInterface, self._config) as sc: result = sc.get_compare_result(compare_id) self.assertEqual(result['plugins']['Software'], self._expected_result()['Software']) self.assertCountEqual( result['plugins']['File_Coverage']['files_in_common'], self._expected_result()['File_Coverage']['files_in_common']) @staticmethod def _expected_result(): return { 'File_Coverage': { 'files_in_common': { 'all': [], 'collapse': False } }, 'Software': { 'Compare Skipped': { 'all': 'Required analysis not present: [\'software_components\', \'software_components\']' } } }
class TestRestFirmware(TestAcceptanceBase): def setUp(self): super().setUp() self.analysis_finished_event = Event() self.elements_finished_analyzing = Value('i', 0) self.db_backend_service = BackEndDbInterface(config=self.config) self._start_backend(post_analysis=self._analysis_callback) self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787' time.sleep(2) # wait for systems to start def tearDown(self): self._stop_backend() self.db_backend_service.shutdown() super().tearDown() def _analysis_callback(self, fo): self.db_backend_service.add_analysis(fo) self.elements_finished_analyzing.value += 1 if self.elements_finished_analyzing.value > 5: self.analysis_finished_event.set() def _rest_upload_firmware(self): testfile_path = os.path.join(get_test_data_dir(), 'container/test.zip') with open(testfile_path, 'rb') as fp: file_content = fp.read() data = { 'binary': standard_b64encode(file_content).decode(), 'file_name': 'test.zip', 'device_name': 'test_device', 'device_class': 'test_class', 'firmware_version': '1.0', 'vendor': 'test_vendor', 'release_date': '01.01.1970', 'tags': '', 'requested_analysis_systems': ['software_components'] } rv = self.test_client.put('/rest/firmware', data=json.dumps(data), follow_redirects=True) self.assertIn(b'"status": 0', rv.data, 'rest upload not successful') self.assertIn(self.test_container_uid.encode(), rv.data, 'uid not found in rest upload reply') def _rest_get_analysis_result(self): rv = self.test_client.get('/rest/firmware/{}'.format(self.test_container_uid), follow_redirects=True) self.assertIn(b'analysis_date', rv.data, 'rest analysis download not successful') self.assertIn(b'software_components', rv.data, 'rest analysis not successful') def _rest_search(self): rv = self.test_client.get('/rest/firmware?query={}'.format(urllib.parse.quote('{"device_class": "test_class"}')), follow_redirects=True) self.assertIn(self.test_container_uid.encode(), rv.data, 'test firmware not found in rest search') def _rest_search_fw_only(self): query = json.dumps({'sha256': self.test_container_uid.split('_')[0]}) rv = self.test_client.get('/rest/firmware?query={}'.format(urllib.parse.quote(query)), follow_redirects=True) self.assertIn(self.test_container_uid.encode(), rv.data, 'test firmware not found in rest search') def _rest_update_analysis_bad_analysis(self): rv = self.test_client.put('/rest/firmware/{}?update={}'.format(self.test_container_uid, urllib.parse.quote('["unknown_system"]')), follow_redirects=True) self.assertIn('Unknown analysis system'.encode(), rv.data, "rest analysis update should break on request of non existing system") def _rest_update_analysis_success(self): rv = self.test_client.put('/rest/firmware/{}?update={}'.format(self.test_container_uid, urllib.parse.quote(json.dumps(['crypto_material']))), follow_redirects=True) self.assertNotIn(b'error_message', rv.data, 'Error on update request') def _rest_check_new_analysis_exists(self): rv = self.test_client.get('/rest/firmware/{}'.format(self.test_container_uid), follow_redirects=True) response_data = json.loads(rv.data.decode()) assert response_data['firmware']['analysis']['crypto_material'] assert response_data['firmware']['analysis']['crypto_material']['analysis_date'] > response_data['firmware']['analysis']['software_components']['analysis_date'] def test_run_from_upload_to_show_analysis_and_search(self): self._rest_upload_firmware() self.analysis_finished_event.wait(timeout=15) self.elements_finished_analyzing.value = 0 self.analysis_finished_event.clear() self._rest_get_analysis_result() self._rest_search() self._rest_search_fw_only() self._rest_update_analysis_bad_analysis() self._rest_update_analysis_success() self.analysis_finished_event.wait(timeout=10) self._rest_check_new_analysis_exists()
class TestStorageDbInterfaceBackend(unittest.TestCase): @classmethod def setUpClass(cls): cls._config = get_config_for_testing(TMP_DIR) cls.mongo_server = MongoMgr(config=cls._config) def setUp(self): self.db_interface = MongoInterfaceCommon(config=self._config) self.db_interface_backend = BackEndDbInterface(config=self._config) self.test_firmware = create_test_firmware() self.test_yara_match = { 'rule': 'OpenSSH', 'tags': [], 'namespace': 'default', 'strings': [(0, '$a', b'OpenSSH')], 'meta': { 'description': 'SSH library', 'website': 'http://www.openssh.com', 'open_source': True, 'software_name': 'OpenSSH' }, 'matches': True } self.test_fo = create_test_file_object() def tearDown(self): self.db_interface.client.drop_database( self._config.get('data_storage', 'main_database')) self.db_interface_backend.shutdown() self.db_interface.shutdown() gc.collect() @classmethod def tearDownClass(cls): cls.mongo_server.shutdown() TMP_DIR.cleanup() def _get_all_firmware_uids(self): uid_list = [] tmp = self.db_interface.firmwares.find() for item in tmp: uid_list.append(item['_id']) return uid_list def test_add_firmware(self): self.db_interface_backend.add_firmware(self.test_firmware) self.assertGreater(len(self._get_all_firmware_uids()), 0, 'No entry added to DB') recoverd_firmware_entry = self.db_interface_backend.firmwares.find_one( ) self.assertAlmostEqual(recoverd_firmware_entry['submission_date'], time(), msg='submission time not set correctly', delta=5.0) def test_add_and_get_firmware(self): self.db_interface_backend.add_firmware(self.test_firmware) result_backend = self.db_interface_backend.get_firmware( self.test_firmware.uid) self.assertIsNotNone(result_backend.binary, 'binary not set in backend result') result_common = self.db_interface.get_firmware(self.test_firmware.uid) self.assertIsNone(result_common.binary, 'binary set in common result') self.assertEqual(result_common.size, 787, 'file size not correct in common') self.assertIsInstance(result_common.tags, dict, 'tag field type not correct') def test_add_and_get_file_object(self): self.db_interface_backend.add_file_object(self.test_fo) result_backend = self.db_interface_backend.get_file_object( self.test_fo.uid) self.assertIsNotNone(result_backend.binary, 'binary not set in backend result') result_common = self.db_interface.get_file_object(self.test_fo.uid) self.assertIsNone(result_common.binary, 'binary set in common result') self.assertEqual(result_common.size, 62, 'file size not correct in common') def test_update_firmware(self): first_dict = { 'stub_plugin': { 'result': 0 }, 'other_plugin': { 'field': 'day' } } second_dict = {'stub_plugin': {'result': 1}} self.test_firmware.processed_analysis = first_dict self.db_interface_backend.add_firmware(self.test_firmware) self.assertEqual( 0, self.db_interface.get_object( self.test_firmware.uid).processed_analysis['stub_plugin'] ['result']) self.test_firmware.processed_analysis = second_dict self.db_interface_backend.add_firmware(self.test_firmware) self.assertEqual( 1, self.db_interface.get_object( self.test_firmware.uid).processed_analysis['stub_plugin'] ['result']) self.assertIn( 'other_plugin', self.db_interface.get_object( self.test_firmware.uid).processed_analysis.keys()) def test_update_file_object(self): first_dict = {'other_plugin': {'result': 0}} second_dict = {'stub_plugin': {'result': 1}} self.test_fo.processed_analysis = first_dict self.test_fo.files_included = {'file a', 'file b'} self.db_interface_backend.add_file_object(self.test_fo) self.test_fo.processed_analysis = second_dict self.test_fo.files_included = {'file b', 'file c'} self.db_interface_backend.add_file_object(self.test_fo) received_object = self.db_interface.get_object(self.test_fo.uid) self.assertEqual( 0, received_object.processed_analysis['other_plugin']['result']) self.assertEqual( 1, received_object.processed_analysis['stub_plugin']['result']) self.assertEqual(3, len(received_object.files_included)) def test_add_and_get_object_including_comment(self): comment, author, date, uid = 'this is a test comment!', 'author', '1473431685', self.test_fo.uid self.test_fo.comments.append({ 'time': str(date), 'author': author, 'comment': comment }) self.db_interface_backend.add_file_object(self.test_fo) retrieved_comment = self.db_interface.get_object(uid).comments[0] self.assertEqual(author, retrieved_comment['author']) self.assertEqual(comment, retrieved_comment['comment']) self.assertEqual(date, retrieved_comment['time']) def test_update_analysis_tag_no_firmware(self): self.db_interface_backend.add_file_object(self.test_fo) tag = {'value': 'yay', 'color': 'default', 'propagate': True} self.db_interface_backend.update_analysis_tags(self.test_fo.uid, plugin_name='dummy', tag_name='some_tag', tag=tag) processed_fo = self.db_interface_backend.get_object(self.test_fo.uid) assert not processed_fo.analysis_tags def test_update_analysis_tag_uid_not_found(self): self.db_interface_backend.update_analysis_tags(self.test_fo.uid, plugin_name='dummy', tag_name='some_tag', tag='should not matter') assert not self.db_interface_backend.get_object(self.test_fo.uid) def test_update_analysis_tag_bad_tag(self): self.db_interface_backend.add_firmware(self.test_firmware) self.db_interface_backend.update_analysis_tags(self.test_firmware.uid, plugin_name='dummy', tag_name='some_tag', tag='bad_tag') processed_firmware = self.db_interface_backend.get_object( self.test_firmware.uid) assert not processed_firmware.analysis_tags def test_update_analysis_tag_success(self): self.db_interface_backend.add_firmware(self.test_firmware) tag = {'value': 'yay', 'color': 'default', 'propagate': True} self.db_interface_backend.update_analysis_tags(self.test_firmware.uid, plugin_name='dummy', tag_name='some_tag', tag=tag) processed_firmware = self.db_interface_backend.get_object( self.test_firmware.uid) assert processed_firmware.analysis_tags assert processed_firmware.analysis_tags['dummy']['some_tag'] == tag def test_add_analysis_firmware(self): self.db_interface_backend.add_object(self.test_firmware) before = self.db_interface_backend.get_object( self.test_firmware.uid).processed_analysis self.test_firmware.processed_analysis['foo'] = {'bar': 5} self.db_interface_backend.add_analysis(self.test_firmware) after = self.db_interface_backend.get_object( self.test_firmware.uid).processed_analysis assert before != after assert 'foo' not in before assert 'foo' in after assert after['foo'] == {'bar': 5} def test_add_analysis_file_object(self): self.db_interface_backend.add_object(self.test_fo) self.test_fo.processed_analysis['foo'] = {'bar': 5} self.db_interface_backend.add_analysis(self.test_fo) analysis = self.db_interface_backend.get_object( self.test_fo.uid).processed_analysis assert 'foo' in analysis assert analysis['foo'] == {'bar': 5} def test_crash_add_analysis(self): with self.assertRaises(RuntimeError): self.db_interface_backend.add_analysis(dict()) with self.assertRaises(AttributeError): self.db_interface_backend._update_analysis(dict(), 'dummy', dict())
class TestStorageDbInterfaceFrontend(unittest.TestCase): @classmethod def setUpClass(cls): cls._config = get_config_for_testing(TMP_DIR) cls.mongo_server = MongoMgr(config=cls._config) def setUp(self): self.db_frontend_interface = FrontEndDbInterface(config=self._config) self.db_backend_interface = BackEndDbInterface(config=self._config) self.test_firmware = create_test_firmware() def tearDown(self): self.db_frontend_interface.shutdown() self.db_backend_interface.client.drop_database( self._config.get('data_storage', 'main_database')) self.db_backend_interface.shutdown() gc.collect() @classmethod def tearDownClass(cls): cls.mongo_server.shutdown() TMP_DIR.cleanup() def test_regression_meta_list(self): assert self.test_firmware.processed_analysis.pop('unpacker') self.db_backend_interface.add_firmware(self.test_firmware) list_of_firmwares = self.db_frontend_interface.get_meta_list() assert 'NOP' in list_of_firmwares.pop()[2] def test_get_meta_list(self): self.db_backend_interface.add_firmware(self.test_firmware) list_of_firmwares = self.db_frontend_interface.get_meta_list() test_output = list_of_firmwares.pop() self.assertEqual(test_output[1], 'test_vendor test_router - 0.1 (Router)', 'Firmware not successfully received') self.assertIsInstance(test_output[2], dict, 'tag field is not a dict') def test_get_meta_list_of_fo(self): test_fo = create_test_file_object() self.db_backend_interface.add_file_object(test_fo) files = self.db_frontend_interface.file_objects.find() meta_list = self.db_frontend_interface.get_meta_list(files) self.assertEqual(meta_list[0][0], test_fo.uid, 'uid of object not correct') self.assertEqual(meta_list[0][3], 0, 'non existing submission date should lead to 0') def test_get_hid_firmware(self): self.db_backend_interface.add_firmware(self.test_firmware) result = self.db_frontend_interface.get_hid(self.test_firmware.uid) self.assertEqual(result, 'test_vendor test_router - 0.1 (Router)', 'fw hid not correct') def test_get_hid_fo(self): test_fo = create_test_file_object(bin_path='get_files_test/testfile2') test_fo.virtual_file_path = { 'a': ['|a|/test_file'], 'b': ['|b|/get_files_test/testfile2'] } self.db_backend_interface.add_file_object(test_fo) result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='b') self.assertEqual(result, '/get_files_test/testfile2', 'fo hid not correct') result = self.db_frontend_interface.get_hid(test_fo.uid) self.assertIsInstance(result, str, 'result is not a string') self.assertEqual(result[0], '/', 'first character not correct if no root_uid set') result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='c') self.assertEqual( result[0], '/', 'first character not correct if invalid root_uid set') def test_get_file_name(self): self.db_backend_interface.add_firmware(self.test_firmware) result = self.db_frontend_interface.get_file_name( self.test_firmware.uid) self.assertEqual(result, 'test.zip', 'name not correct') def test_get_hid_invalid_uid(self): result = self.db_frontend_interface.get_hid('foo') self.assertEqual(result, '', 'invalid uid should result in empty string') def test_get_firmware_attribute_list(self): self.db_backend_interface.add_firmware(self.test_firmware) self.assertEqual(self.db_frontend_interface.get_device_class_list(), ['Router']) self.assertEqual(self.db_frontend_interface.get_vendor_list(), ['test_vendor']) self.assertEqual( self.db_frontend_interface.get_firmware_attribute_list( 'device_name', { 'vendor': 'test_vendor', 'device_class': 'Router' }), ['test_router']) self.assertEqual( self.db_frontend_interface.get_firmware_attribute_list('version'), ['0.1']) self.assertEqual(self.db_frontend_interface.get_device_name_dict(), {'Router': { 'test_vendor': ['test_router'] }}) def test_get_data_for_nice_list(self): uid_list = [self.test_firmware.uid] self.db_backend_interface.add_firmware(self.test_firmware) nice_list_data = self.db_frontend_interface.get_data_for_nice_list( uid_list, uid_list[0]) self.assertEqual( sorted([ 'size', 'virtual_file_paths', 'uid', 'mime-type', 'files_included' ]), sorted(nice_list_data[0].keys())) self.assertEqual(nice_list_data[0]['uid'], self.test_firmware.uid) def test_generic_search(self): self.db_backend_interface.add_firmware(self.test_firmware) # str input result = self.db_frontend_interface.generic_search( '{"file_name": "test.zip"}') self.assertEqual(result, [self.test_firmware.uid], 'Firmware not successfully received') # dict input result = self.db_frontend_interface.generic_search( {'file_name': 'test.zip'}) self.assertEqual(result, [self.test_firmware.uid], 'Firmware not successfully received') def test_all_uids_found_in_database(self): self.db_backend_interface.client.drop_database( self._config.get('data_storage', 'main_database')) uid_list = [self.test_firmware.uid] self.assertFalse( self.db_frontend_interface.all_uids_found_in_database(uid_list)) self.db_backend_interface.add_firmware(self.test_firmware) self.assertTrue( self.db_frontend_interface.all_uids_found_in_database( [self.test_firmware.uid])) def test_get_x_last_added_firmwares(self): self.assertEqual(self.db_frontend_interface.get_last_added_firmwares(), [], 'empty db should result in empty list') test_fw_one = create_test_firmware(device_name='fw_one') self.db_backend_interface.add_firmware(test_fw_one) test_fw_two = create_test_firmware(device_name='fw_two', bin_path='container/test.7z') self.db_backend_interface.add_firmware(test_fw_two) test_fw_three = create_test_firmware(device_name='fw_three', bin_path='container/test.cab') self.db_backend_interface.add_firmware(test_fw_three) result = self.db_frontend_interface.get_last_added_firmwares(limit_x=2) self.assertEqual(len(result), 2, 'Number of results should be 2') self.assertEqual(result[0][0], test_fw_three.uid, 'last firmware is not first entry') self.assertEqual(result[1][0], test_fw_two.uid, 'second last firmware is not the second entry') def test_generate_file_tree_level(self): parent_fw = create_test_firmware() child_fo = create_test_file_object() child_fo.processed_analysis['file_type'] = {'mime': 'sometype'} uid = parent_fw.uid child_fo.virtual_file_path = { uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)] } parent_fw.files_included = {child_fo.uid} self.db_backend_interface.add_object(parent_fw) self.db_backend_interface.add_object(child_fo) for node in self.db_frontend_interface.generate_file_tree_level( uid, uid): self.assertIsInstance(node, FileTreeNode) self.assertEqual(node.name, parent_fw.file_name) self.assertTrue(node.has_children) for node in self.db_frontend_interface.generate_file_tree_level( child_fo.uid, uid): self.assertIsInstance(node, FileTreeNode) self.assertEqual(node.name, 'folder') self.assertTrue(node.has_children) virtual_grand_child = node.get_list_of_child_nodes()[0] self.assertEqual(virtual_grand_child.type, 'sometype') self.assertFalse(virtual_grand_child.has_children) self.assertEqual(virtual_grand_child.name, child_fo.file_name) def test_get_number_of_total_matches(self): parent_fw = create_test_firmware() child_fo = create_test_file_object() uid = parent_fw.uid child_fo.virtual_file_path = { uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)] } self.db_backend_interface.add_object(parent_fw) self.db_backend_interface.add_object(child_fo) query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format( uid, child_fo.uid) self.assertEqual( self.db_frontend_interface.get_number_of_total_matches( query, only_parent_firmwares=False), 2) self.assertEqual( self.db_frontend_interface.get_number_of_total_matches( query, only_parent_firmwares=True), 1) def test_get_other_versions_of_firmware(self): parent_fw1 = create_test_firmware(version='1') self.db_backend_interface.add_object(parent_fw1) parent_fw2 = create_test_firmware(version='2', bin_path='container/test.7z') self.db_backend_interface.add_object(parent_fw2) parent_fw3 = create_test_firmware(version='3', bin_path='container/test.cab') self.db_backend_interface.add_object(parent_fw3) other_versions = self.db_frontend_interface.get_other_versions_of_firmware( parent_fw1) self.assertEqual(len(other_versions), 2, 'wrong number of other versions') self.assertIn({'_id': parent_fw2.uid, 'version': '2'}, other_versions) self.assertIn({'_id': parent_fw3.uid, 'version': '3'}, other_versions) other_versions = self.db_frontend_interface.get_other_versions_of_firmware( parent_fw2) self.assertIn({'_id': parent_fw3.uid, 'version': '3'}, other_versions) def test_get_specific_fields_for_multiple_entries(self): test_fw_1 = create_test_firmware(device_name='fw_one', vendor='test_vendor_one') self.db_backend_interface.add_firmware(test_fw_1) test_fw_2 = create_test_firmware(device_name='fw_two', vendor='test_vendor_two', bin_path='container/test.7z') self.db_backend_interface.add_firmware(test_fw_2) test_fo = create_test_file_object() self.db_backend_interface.add_file_object(test_fo) test_uid_list = [test_fw_1.uid, test_fw_2.uid] result = list( self.db_frontend_interface. get_specific_fields_for_multiple_entries(uid_list=test_uid_list, field_dict={ 'vendor': 1, 'device_name': 1 })) assert len(result) == 2 assert all( set(entry.keys()) == {'_id', 'vendor', 'device_name'} for entry in result) result_uids = [entry['_id'] for entry in result] assert all(uid in result_uids for uid in test_uid_list) test_uid_list = [test_fw_1.uid, test_fo.uid] result = list( self.db_frontend_interface. get_specific_fields_for_multiple_entries( uid_list=test_uid_list, field_dict={'virtual_file_path': 1})) assert len(result) == 2 assert all( set(entry.keys()) == {'_id', 'virtual_file_path'} for entry in result) result_uids = [entry['_id'] for entry in result] assert all(uid in result_uids for uid in test_uid_list) def test_find_missing_files(self): test_fw_1 = create_test_firmware() test_fw_1.files_included.add('uid1234') self.db_backend_interface.add_firmware(test_fw_1) missing_files = self.db_frontend_interface.find_missing_files() assert test_fw_1.uid in missing_files assert missing_files[test_fw_1.uid] == {'uid1234'} test_fo = create_test_file_object() test_fo.uid = 'uid1234' self.db_backend_interface.add_file_object(test_fo) missing_files = self.db_frontend_interface.find_missing_files() assert missing_files == {} def test_find_missing_analyses(self): test_fw_1 = create_test_firmware() test_fo = create_test_file_object() test_fw_1.files_included.add(test_fo.uid) test_fo.virtual_file_path = {test_fw_1.uid: ['|foo|bar|']} self.db_backend_interface.add_firmware(test_fw_1) self.db_backend_interface.add_file_object(test_fo) missing_analyses = self.db_frontend_interface.find_missing_analyses() assert missing_analyses == {} test_fw_1.processed_analysis['foobar'] = {'foo': 'bar'} self.db_backend_interface.add_analysis(test_fw_1) missing_analyses = self.db_frontend_interface.find_missing_analyses() assert test_fw_1.uid in missing_analyses assert missing_analyses[test_fw_1.uid] == {test_fo.uid}
class TestRestFirmware(TestAcceptanceBase): def setUp(self): super().setUp() self.analysis_finished_event = Event() self.elements_finished_analyzing = Value('i', 0) self.db_backend_service = BackEndDbInterface(config=self.config) self._start_backend(post_analysis=self._analysis_callback) self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787' time.sleep(2) # wait for systems to start def tearDown(self): self._stop_backend() self.db_backend_service.shutdown() super().tearDown() def _analysis_callback(self, fo): self.db_backend_service.add_analysis(fo) self.elements_finished_analyzing.value += 1 if self.elements_finished_analyzing.value == 4 * 3: # container including 3 files times 3 plugins self.analysis_finished_event.set() def _rest_upload_firmware(self): data = get_firmware_for_rest_upload_test() rv = self.test_client.put('/rest/firmware', json=data, follow_redirects=True) assert b'"status": 0' in rv.data, 'rest upload not successful' assert self.test_container_uid.encode( ) in rv.data, 'uid not found in rest upload reply' def _rest_get_analysis_result(self): rv = self.test_client.get(f'/rest/firmware/{self.test_container_uid}', follow_redirects=True) assert b'analysis_date' in rv.data, 'rest analysis download not successful' assert b'software_components' in rv.data, 'rest analysis not successful' assert b'"device_part": "test_part' in rv.data, 'device part not present' def _rest_search(self): query = urllib.parse.quote('{"device_class": "test_class"}') rv = self.test_client.get(f'/rest/firmware?query={query}', follow_redirects=True) assert self.test_container_uid.encode( ) in rv.data, 'test firmware not found in rest search' def _rest_search_fw_only(self): query = json.dumps({'sha256': self.test_container_uid.split('_')[0]}) rv = self.test_client.get( f'/rest/firmware?query={urllib.parse.quote(query)}', follow_redirects=True) assert self.test_container_uid.encode( ) in rv.data, 'test firmware not found in rest search' def _rest_update_analysis_bad_analysis(self): query = urllib.parse.quote('["unknown_system"]') rv = self.test_client.put( f'/rest/firmware/{self.test_container_uid}?update={query}', follow_redirects=True) assert 'Unknown analysis system'.encode( ) in rv.data, 'rest analysis update should break on request of non existing system' def _rest_update_analysis_success(self): update = urllib.parse.quote(json.dumps(['crypto_material'])) rv = self.test_client.put( f'/rest/firmware/{self.test_container_uid}?update={update}', follow_redirects=True) assert b'error_message' not in rv.data, 'Error on update request' def _rest_check_new_analysis_exists(self): rv = self.test_client.get(f'/rest/firmware/{self.test_container_uid}', follow_redirects=True) response_data = json.loads(rv.data.decode()) assert response_data['firmware']['analysis']['crypto_material'] assert response_data['firmware']['analysis']['crypto_material'][ 'analysis_date'] > response_data['firmware']['analysis'][ 'software_components']['analysis_date'] def test_run_from_upload_to_show_analysis_and_search(self): self._rest_upload_firmware() self.analysis_finished_event.wait(timeout=15) self.elements_finished_analyzing.value = 4 * 2 # only one plugin to update so we offset with 4 times 2 plugins self.analysis_finished_event.clear() self._rest_get_analysis_result() self._rest_search() self._rest_search_fw_only() self._rest_update_analysis_bad_analysis() self._rest_update_analysis_success() self.analysis_finished_event.wait(timeout=10) self._rest_check_new_analysis_exists()