def test_add_file_object_to_file_object(): root = create_test_firmware() child_fo = create_test_file_object() root.add_included_file(child_fo) grandchild_fo = create_test_file_object( bin_path='get_files_test/testfile2') child_fo.add_included_file(grandchild_fo) assert grandchild_fo.virtual_file_path[root.uid][0] == join_virtual_path( root.uid, child_fo.uid, grandchild_fo.file_path)
def test_find_failed_analyses_with_multiple_files(self): test_fo_1 = create_test_file_object() test_fo_1.processed_analysis.update({'foo': {'failed': 'some reason'}}) test_fo_2 = create_test_file_object(bin_path='container/test.7z') test_fo_2.processed_analysis.update({'foo': {'failed': 'no reason'}}) assert test_fo_1.uid != test_fo_2.uid, 'files should not be the same' self.db_backend_interface.add_file_object(test_fo_1) self.db_backend_interface.add_file_object(test_fo_2) failed_analyses = self.db_frontend_interface.find_failed_analyses() assert failed_analyses, 'should not be empty' assert list(failed_analyses) == ['foo'] assert len(failed_analyses['foo']) == 2 assert test_fo_1.uid in failed_analyses['foo'] and test_fo_2.uid in failed_analyses['foo']
def test_generate_file_tree_level(self): parent_fw = create_test_firmware() child_fo = create_test_file_object() child_fo.processed_analysis['file_type'] = {'mime': 'sometype'} uid = parent_fw.uid child_fo.virtual_file_path = { uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)] } parent_fw.files_included = {child_fo.uid} self.db_backend_interface.add_object(parent_fw) self.db_backend_interface.add_object(child_fo) for node in self.db_frontend_interface.generate_file_tree_level( uid, uid): assert isinstance(node, FileTreeNode) assert node.name == parent_fw.file_name assert node.has_children for node in self.db_frontend_interface.generate_file_tree_level( child_fo.uid, uid): assert isinstance(node, FileTreeNode) assert node.name == 'folder' assert node.has_children virtual_grand_child = node.get_list_of_child_nodes()[0] assert virtual_grand_child.type == 'sometype' assert not virtual_grand_child.has_children assert virtual_grand_child.name == child_fo.file_name
def test_get_results_from_parent_fos__multiple_vfps_in_one_fw(self): fw = create_test_firmware() fo = create_test_file_object() file_names = ['file_a', 'file_b', 'file_c'] fw.processed_analysis[AnalysisPlugin.NAME] = { 'files': {b64_encode(f): { 'result': 'value' } for f in file_names} } vfp = fo.virtual_file_path['some_uid'] = [] for f in file_names: vfp.append('some_uid|{}|/{}'.format(fw.uid, f)) results = {} routes.FsMetadataRoutesDbInterface.get_results_from_parent_fos( fw, fo, results) assert results is not None assert results != {}, 'result should not be empty' assert len(results) == 3, 'wrong number of results' assert all(f in results for f in file_names), 'files missing from result' assert 'result' in results[file_names[0]], 'analysis result is missing' assert results[file_names[0]][ 'result'] == 'value', 'wrong value of analysis result'
def setUp(self): self._config = get_config_for_testing(TMP_DIR) self._config.set('data_storage', 'report_threshold', '32') self._config.set('data_storage', 'sanitize_database', 'tmp_sanitize') self.mongo_server = MongoMgr(config=self._config) self.db_interface = MongoInterfaceCommon(config=self._config) self.db_interface_backend = BackEndDbInterface(config=self._config) self.test_firmware = create_test_firmware() self.test_yara_match = { 'rule': 'OpenSSH', 'tags': [], 'namespace': 'default', 'strings': [(0, '$a', b'OpenSSH')], 'meta': { 'description': 'SSH library', 'website': 'http://www.openssh.com', 'open_source': True, 'software_name': 'OpenSSH' }, 'matches': True } self.test_fo = create_test_file_object()
def test_get_results_from_parent_fos(self): fw = create_test_firmware() fo = create_test_file_object() file_name = 'folder/file' encoded_name = b64_encode(file_name) fw.processed_analysis[AnalysisPlugin.NAME] = { 'files': { encoded_name: { 'result': 'value' } } } fo.virtual_file_path['some_uid'] = [ 'some_uid|{}|/{}'.format(fw.uid, file_name) ] results = {} routes.FsMetadataRoutesDbInterface.get_results_from_parent_fos( fw, fo, results) assert results != {}, 'result should not be empty' assert file_name in results, 'files missing from result' assert 'parent_uid' in results[ file_name], 'parent uid missing in result' assert 'result' in results[file_name], 'analysis result is missing' assert results[file_name][ 'result'] == 'value', 'wrong value of analysis result'
def test_get_file_object_number(self): result = self.db_interface.get_file_object_number() self.assertEqual(result, 0) self.db_interface_backend.add_file_object(self.test_fo) result = self.db_interface.get_file_object_number( query={}, zero_on_empty_query=False) self.assertEqual(result, 1) result = self.db_interface.get_file_object_number( query={'_id': self.test_fo.uid}) self.assertEqual(result, 1) result = self.db_interface.get_file_object_number( query=json.dumps({'_id': self.test_fo.uid})) self.assertEqual(result, 1) result = self.db_interface.get_file_object_number( query={}, zero_on_empty_query=True) self.assertEqual(result, 0) result = self.db_interface.get_file_object_number( query='{}', zero_on_empty_query=True) self.assertEqual(result, 0) test_fo_2 = create_test_file_object( bin_path='get_files_test/testfile2') self.db_interface_backend.add_file_object(test_fo_2) result = self.db_interface.get_file_object_number( query={}, zero_on_empty_query=False) self.assertEqual(result, 2) result = self.db_interface.get_file_object_number( query={'_id': self.test_fo.uid}) self.assertEqual(result, 1)
def test_get_meta_list_of_fo(self): test_fo = create_test_file_object() self.db_backend_interface.add_file_object(test_fo) files = self.db_frontend_interface.file_objects.find() meta_list = self.db_frontend_interface.get_meta_list(files) self.assertEqual(meta_list[0][0], test_fo.uid, 'uid of object not correct') self.assertEqual(meta_list[0][3], 0, 'non existing submission date should lead to 0')
def test_imphash(): fo = create_test_file_object( bin_path=str(Path(get_test_data_dir(), 'test_executable'))) fo.processed_analysis = {'file_type': {'mime': 'application/x-executable'}} imphash = get_imphash(fo) assert isinstance(imphash, str), 'imphash should be a string' assert len(imphash) == 32, 'imphash does not seem to be an md5'
def test_process_object_this_file(stub_plugin): test_file = create_test_file_object(bin_path=str(PYLINT_TEST_FILE)) stub_plugin.process_object(test_file) result = test_file.processed_analysis[stub_plugin.NAME] assert result['full'] assert result['full'][0]['type'] == 'warning' assert result['full'][0]['symbol'] == 'unused-import'
def test_generate_file_tree_node(self): parent_fw = create_test_firmware() child_fo = create_test_file_object() child_fo.processed_analysis['file_type'] = {'mime': 'sometype'} uid = parent_fw.get_uid() child_fo.virtual_file_path = { uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)] } parent_fw.files_included = {child_fo.get_uid()} self.db_backend_interface.add_object(parent_fw) self.db_backend_interface.add_object(child_fo) for node in self.db_frontend_interface.generate_file_tree_node( uid, uid): self.assertIsInstance(node, FileTreeNode) self.assertEqual(node.name, parent_fw.file_name) self.assertTrue(node.has_children) for node in self.db_frontend_interface.generate_file_tree_node( child_fo.get_uid(), uid): self.assertIsInstance(node, FileTreeNode) self.assertEqual(node.name, 'folder') self.assertTrue(node.has_children) virtual_grand_child = node.get_list_of_child_nodes()[0] self.assertEqual(virtual_grand_child.type, 'sometype') self.assertFalse(virtual_grand_child.has_children) self.assertEqual(virtual_grand_child.name, child_fo.file_name)
def test_get_executable_stats(self): for i, file_str in enumerate([ 'ELF 64-bit LSB executable, x86-64, dynamically linked, for GNU/Linux 2.6.32, not stripped', 'ELF 32-bit MSB executable, MIPS, MIPS32 rel2 version 1 (SYSV), statically linked, not stripped', 'ELF 64-bit LSB executable, x86-64, (SYSV), corrupted section header size', 'ELF 64-bit LSB executable, aarch64, dynamically linked, stripped', 'ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, stripped' ]): fo = create_test_file_object() fo.processed_analysis['file_type'] = {'full': file_str} fo.uid = str(i) self.db_backend_interface.add_file_object(fo) stats = self.updater.get_executable_stats().get('executable_stats') expected = [('big endian', 1, 0.25), ('little endian', 3, 0.75), ('stripped', 1, 0.25), ('not stripped', 2, 0.5), ('32-bit', 1, 0.25), ('64-bit', 3, 0.75), ('dynamically linked', 2, 0.5), ('statically linked', 1, 0.25), ('section info missing', 1, 0.25)] for (expected_label, expected_count, expected_percentage), (label, count, percentage, _) in zip(expected, stats): assert label == expected_label assert count == expected_count assert percentage == expected_percentage
def test_get_specific_fields_for_multiple_entries(self): test_fw_1 = create_test_firmware(device_name='fw_one', vendor='test_vendor_one') self.db_backend_interface.add_firmware(test_fw_1) test_fw_2 = create_test_firmware(device_name='fw_two', vendor='test_vendor_two', bin_path='container/test.7z') self.db_backend_interface.add_firmware(test_fw_2) test_fo = create_test_file_object() self.db_backend_interface.add_file_object(test_fo) test_uid_list = [test_fw_1.uid, test_fw_2.uid] result = list(self.db_frontend_interface.get_specific_fields_for_multiple_entries( uid_list=test_uid_list, field_dict={'vendor': 1, 'device_name': 1} )) assert len(result) == 2 assert all(set(entry.keys()) == {'_id', 'vendor', 'device_name'} for entry in result) result_uids = [entry['_id'] for entry in result] assert all(uid in result_uids for uid in test_uid_list) test_uid_list = [test_fw_1.uid, test_fo.uid] result = list(self.db_frontend_interface.get_specific_fields_for_multiple_entries( uid_list=test_uid_list, field_dict={'virtual_file_path': 1} )) assert len(result) == 2 assert all(set(entry.keys()) == {'_id', 'virtual_file_path'} for entry in result) result_uids = [entry['_id'] for entry in result] assert all(uid in result_uids for uid in test_uid_list)
def test_add_file_object_to_firmware(): root = create_test_firmware() child_fo = create_test_file_object() root.add_included_file(child_fo) assert root.uid in child_fo.virtual_file_path.keys( ), 'no virtual file path for root available' assert child_fo.virtual_file_path[root.uid][0] == join_virtual_path( root.uid, child_fo.file_path), 'virtual file path not correct'
def test_rest_get_failed_analyses(self): test_fo = create_test_file_object() test_fo.processed_analysis['some_analysis'] = {'failed': 'oops'} self.db_backend.add_file_object(test_fo) response = json.loads(self.test_client.get('/rest/missing', follow_redirects=True).data.decode()) assert 'failed_analyses' in response assert 'some_analysis' in response['failed_analyses'] assert test_fo.uid in response['failed_analyses']['some_analysis']
def get_object(self, uid): if uid == self.fw.uid: return self.fw if uid == 'foo': return self.fo if uid == 'bar': fo = create_test_file_object() fo.virtual_file_path = {'some_uid': ['a|b|c']} return fo
def test_find_strings(self): fo = create_test_file_object() fo.processed_analysis['printable_strings'] = dict(strings=['reasonable', 'still_reasonable', 'n123ot\'(§rea\'§&son##+able']) fo = self.analysis_plugin.process_object(fo) results = fo.processed_analysis[self.PLUGIN_NAME] self.assertTrue(isinstance(results, dict), 'Result of wrong type') self.assertTrue(results['string_eval'] == ['still_reasonable', 'reasonable', 'n123ot\'(§rea\'§&son##+able'])
def __init__(self): self.fw = create_test_firmware() self.fo = create_test_file_object() self.fo.processed_analysis['file_hashes'] = { 'ssdeep': get_ssdeep(self.fo.binary) } self.fw.add_included_file(self.fo) self.fw.processed_analysis['file_hashes'] = { 'ssdeep': get_ssdeep(self.fw.binary) }
def test_unpack_status_packed_file(self): test_fo_packed = create_test_file_object(bin_path='container/test.7z') test_fo_packed.processed_analysis['unpacker'] = {} self.unpacker.get_unpack_status(test_fo_packed, []) result = test_fo_packed.processed_analysis['unpacker'] self.assertGreater(result['entropy'], 0.7, 'entropy not valid') self.assertEqual(result['summary'], ['packed'], '7z file should be packed') self.unpacker.VALID_COMPRESSED_FILE_TYPES = ['application/x-7z-compressed'] self.unpacker.get_unpack_status(test_fo_packed, []) self.assertEqual(test_fo_packed.processed_analysis['unpacker']['summary'], ['unpacked'], 'Unpacking Whitelist does not work')
def test_rest_get_orphaned_objects(self): test_fo = create_test_file_object() test_fo.parent_firmware_uids = ['missing_uid'] self.db_backend.add_file_object(test_fo) response = json.loads(self.test_client.get('/rest/missing', follow_redirects=True).data.decode()) assert 'orphaned_objects' in response assert response['orphaned_objects'] == { 'missing_uid': ['d558c9339cb967341d701e3184f863d3928973fccdc1d96042583730b5c7b76a_62'] }
def test_get_number_of_total_matches(self): parent_fw = create_test_firmware() child_fo = create_test_file_object() uid = parent_fw.uid child_fo.virtual_file_path = {uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]} self.db_backend_interface.add_object(parent_fw) self.db_backend_interface.add_object(child_fo) query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(uid, child_fo.uid) self.assertEqual(self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=False), 2) self.assertEqual(self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True), 1)
def test_find_failed_analyses_with_multiple_analyses(self): test_fo_1 = create_test_file_object() test_fo_1.processed_analysis.update({'foo': {'failed': 'some reason'}, 'bar': {'failed': 'another reason'}}) self.db_backend_interface.add_file_object(test_fo_1) failed_analyses = self.db_frontend_interface.find_failed_analyses() assert failed_analyses, 'should not be empty' assert sorted(failed_analyses) == ['bar', 'foo'] assert len(failed_analyses['foo']) == 1 and len(failed_analyses['bar']) == 1 assert test_fo_1.uid in failed_analyses['foo']
def test_rest_download_valid(self): test_file_object = create_test_file_object() self.db_backend.add_file_object(test_file_object) rv = self.test_client.get('/rest/file_object/{}'.format( test_file_object.uid), follow_redirects=True) assert b'hid' in rv.data assert b'size' in rv.data
def test_process_object_this_file(stub_plugin, monkeypatch): test_file = create_test_file_object(bin_path=str(PYLINT_TEST_FILE)) monkeypatch.setattr( 'storage.fsorganizer.FSOrganizer.generate_path_from_uid', lambda _self, _: test_file.file_path) stub_plugin.process_object(test_file) result = test_file.processed_analysis[stub_plugin.NAME] assert result['full'] assert result['full'][0]['type'] == 'warning' assert result['full'][0]['symbol'] == 'unused-import'
def test_get_parent_uids_from_virtual_path(self): fo = create_test_file_object() fo.virtual_file_path = {'fw_uid': ['fw_uid']} assert len( plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo)) == 0 fo.virtual_file_path = {'some_UID': ['|uid1|uid2|/folder_1/some_file']} assert 'uid2' in plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo) fo.virtual_file_path = { 'some_UID': [ '|uid1|uid2|/folder_1/some_file', '|uid1|uid2|/folder_2/some_file' ] } result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo) assert 'uid2' in result assert len(result) == 1 fo.virtual_file_path = { 'uid1': ['|uid1|uid2|/folder_1/some_file', '|uid1|uid3|/some_file'] } result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo) assert 'uid2' in result assert 'uid3' in result assert len(result) == 2 fo.virtual_file_path = { 'uid1': ['|uid1|uid2|/folder_1/some_file'], 'other_UID': ['|other_UID|uid2|/folder_2/some_file'] } result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo) assert 'uid2' in result assert len(result) == 1 fo.virtual_file_path = { 'uid1': ['|uid1|uid2|/folder_1/some_file'], 'other_UID': ['|other_UID|uid3|/folder_2/some_file'] } result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo) assert 'uid2' in result assert 'uid3' in result assert len(result) == 2 fo.virtual_file_path = {} assert len( plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path( fo)) == 0
def test_process_object(self): fo = create_test_file_object() fo.processed_analysis['ip_and_uri_finder'] = { 'summary': ['1.2.3.4', 'www.example.com', 'www.interesting.receive.org'] } self.analysis_plugin.process_object(fo) assert self.PLUGIN_NAME in fo.processed_analysis assert fo.processed_analysis[self.PLUGIN_NAME]['summary'] == [ 'www.interesting.receive.org' ]
def test_add_file_object_path_already_present(): root = create_test_firmware() child = create_test_file_object() child.virtual_file_path = { root.uid: ['{}|some/known/path'.format(root.uid)] } root.add_included_file(child) assert len(child.virtual_file_path.keys() ) == 1, 'there should be just one root object' assert len(child.virtual_file_path[ root.uid]) == 1, 'number of paths should be one'
def test_get_hid_fo(self): test_fo = create_test_file_object(bin_path='get_files_test/testfile2') test_fo.virtual_file_path = {'a': ['|a|/test_file'], 'b': ['|b|/get_files_test/testfile2']} self.db_backend_interface.add_file_object(test_fo) result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='b') self.assertEqual(result, '/get_files_test/testfile2', 'fo hid not correct') result = self.db_frontend_interface.get_hid(test_fo.uid) self.assertIsInstance(result, str, 'result is not a string') self.assertEqual(result[0], '/', 'first character not correct if no root_uid set') result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='c') self.assertEqual(result[0], '/', 'first character not correct if invalid root_uid set')
def test_get_number_of_total_matches(self): parent_fw = create_test_firmware() child_fo = create_test_file_object() uid = parent_fw.uid child_fo.parent_firmware_uids = [uid] self.db_backend_interface.add_object(parent_fw) self.db_backend_interface.add_object(child_fo) query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(uid, child_fo.uid) assert self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=False, inverted=False) == 2 assert self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True, inverted=False) == 1 assert self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True, inverted=True) == 0
def test_get_complete_object_including_all_summaries(self): self.db_interface_backend.report_threshold = 1024 test_file = create_test_file_object() self.test_firmware.add_included_file(test_file) self.db_interface_backend.add_firmware(self.test_firmware) self.db_interface_backend.add_file_object(test_file) tmp = self.db_interface.get_complete_object_including_all_summaries(self.test_firmware.uid) self.assertIsInstance(tmp, Firmware, 'wrong type') self.assertIn('summary', tmp.processed_analysis['dummy'].keys(), 'summary not found in processed analysis') self.assertIn('sum a', tmp.processed_analysis['dummy']['summary'], 'summary of original file not included') self.assertIn('file exclusive sum b', tmp.processed_analysis['dummy']['summary'], 'summary of included file not found')