def test_unpack_analyse_and_compare(self): test_fw_1 = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) test_fw_1.release_date = '2017-01-01' test_fw_2 = Firmware( file_path='{}/regression_one'.format(get_test_data_dir())) test_fw_2.release_date = '2017-01-01' self._unpack_scheduler.add_task(test_fw_1) self._unpack_scheduler.add_task(test_fw_2) self.analysis_finished_event.wait(timeout=20) compare_id = normalize_compare_id(';'.join( [fw.uid for fw in [test_fw_1, test_fw_2]])) self.assertIsNone( self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error') self.compare_finished_event.wait(timeout=10) with ConnectTo(CompareDbInterface, self._config) as sc: result = sc.get_compare_result(compare_id) self.assertEqual(result['plugins']['Software'], self._expected_result()['Software']) self.assertCountEqual( result['plugins']['File_Coverage']['files_in_common'], self._expected_result()['File_Coverage']['files_in_common'])
def test_execute_yara_search_for_single_file(self): test_rule_path = path.join(get_test_data_dir(), 'yara_binary_search_test_rule') result = self.yara_binary_scanner._execute_yara_search( test_rule_path, target_path=path.join(get_test_data_dir(), TEST_FILE_1, TEST_FILE_1)) self.assertTrue('test_rule' in result)
def test_get_temp_dir_path(caplog): assert get_temp_dir_path() == '/tmp' assert get_temp_dir_path(MockConfig( get_test_data_dir())) == get_test_data_dir() not_a_dir = str(Path(get_test_data_dir()) / '__init__.py') with caplog.at_level(logging.WARNING): assert get_temp_dir_path(MockConfig(not_a_dir)) == '/tmp' assert 'TempDir path does not exist and could not be created' in caplog.messages[ 0]
def test_file_is_zero(self): self.assertTrue( file_is_empty('{}/zero_byte'.format(get_test_data_dir())), 'file is empty but stated differently') self.assertFalse( file_is_empty('{}/get_files_test/testfile1'.format( get_test_data_dir())), 'file not empty but stated differently') self.assertFalse( file_is_empty(os.path.join(get_test_data_dir(), 'broken_link')), 'Broken link is not empty')
def test_file_is_locked(self): assert not self.unpacker.db_interface.check_unpacking_lock( self.test_fo.uid) file_paths = [ '{}/get_files_test/testfile1'.format(get_test_data_dir()) ] self.unpacker.generate_and_store_file_objects(file_paths, get_test_data_dir(), self.test_fo) assert self.unpacker.db_interface.check_unpacking_lock( self.test_fo.uid)
def test_dont_store_zero_file(self): file_pathes = [ Path(get_test_data_dir(), 'files', 'zero_byte'), Path(get_test_data_dir(), 'files', 'get_files_test', 'testfile1') ] file_objects = self.unpacker.generate_and_store_file_objects( file_pathes, get_test_data_dir(), self.test_fo) file_objects = make_list_from_dict(file_objects) self.assertEqual(len(file_objects), 1, 'number of objects not correct') self.assertEqual(file_objects[0].file_name, 'testfile1', 'wrong object created') parent_uid = self.test_fo.uid self.assertIn('|{}|/get_files_test/testfile1'.format(parent_uid), file_objects[0].virtual_file_path[self.test_fo.uid])
class argument_mock(): config_file = get_test_data_dir() + '/load_cfg_test' log_file = '/log/file/path' log_level = 'DEBUG' silent = False debug = False
def test_process_next_analysis_unknown_plugin(self): test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1')) test_fw.scheduled_analysis = ['unknown_plugin'] with mock_spy(self.sched, '_start_or_skip_analysis') as spy: self.sched.process_next_analysis(test_fw) assert not spy.was_called(), 'unknown plugin should simply be skipped'
def get_file_name_and_path(self, uid): if uid != 'foo': return { 'file_path': os.path.join(get_test_data_dir(), 'container/test.zip') } return None
def test_set_binary(): binary = get_binary_from_file('{}/get_files_test/testfile1'.format( get_test_data_dir())) md5 = 'e802ca22f6cd2d9357cf3da1d191879e' firmware = Firmware() firmware.set_binary(binary) assert firmware.md5 == md5
class TestUnpackerCoreMain(TestUnpackerBase): test_file_path = str(Path(get_test_data_dir()) / 'container/test.zip') def main_unpack_check(self, test_object, number_unpacked_files, first_unpacker): extracted_files = self.unpacker.unpack(test_object) assert len(test_object.files_included ) == number_unpacked_files, 'not all files added to parent' assert len( extracted_files) == number_unpacked_files, 'not all files found' assert test_object.processed_analysis['unpacker'][ 'plugin_used'] == first_unpacker, 'Wrong plugin in Meta' assert test_object.processed_analysis['unpacker'][ 'number_of_unpacked_files'] == number_unpacked_files, 'Number of unpacked files wrong in Meta' self.check_depths_of_children(test_object, extracted_files) @staticmethod def check_depths_of_children(parent, extracted_files): for item in extracted_files: assert item.depth == parent.depth + 1, 'depth of child not correct' def test_main_unpack_function(self): test_file = FileObject(file_path=self.test_file_path) self.main_unpack_check(test_file, 3, '7z') def test_unpacking_depth_reached(self): test_file = FileObject(file_path=self.test_file_path) test_file.depth = 10 self.unpacker.unpack(test_file) assert 'unpacker' in test_file.processed_analysis assert 'maximum unpacking depth was reached' in test_file.processed_analysis[ 'unpacker']['info']
def test_imphash(): fo = create_test_file_object( bin_path=str(Path(get_test_data_dir(), 'test_executable'))) fo.processed_analysis = {'file_type': {'mime': 'application/x-executable'}} imphash = get_imphash(fo) assert isinstance(imphash, str), 'imphash should be a string' assert len(imphash) == 32, 'imphash does not seem to be an md5'
def test_process_object(self): test_file = FileObject(file_path=os.path.join(get_test_data_dir(), 'yara_test_file')) test_file.processed_analysis.update({self.PLUGIN_NAME: []}) processed_file = self.analysis_plugin.process_object(test_file) results = processed_file.processed_analysis[self.PLUGIN_NAME] assert len(results) == 2, 'not all matches found' assert 'testRule' in results, 'testRule match not found' assert results['summary'] == ['testRule']
def test_create_firmware_from_file(): test_object = Firmware( file_path='{}/test_data_file.bin'.format(get_test_data_dir())) assert test_object.device_name is None assert test_object.size == 19 assert test_object.binary == b'test string in file' assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8' assert test_object.file_name == 'test_data_file.bin'
def test_get_file_from_binary(self): file_path = '{}/test_data_file.bin'.format(get_test_data_dir()) test_object = FileObject(file_path=file_path) assert test_object.size == 19, 'correct size' assert test_object.binary == b'test string in file', 'correct binary data' assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8', 'correct sha256' assert test_object.file_name == 'test_data_file.bin', 'correct file name' assert test_object.file_path == file_path, 'correct file path'
def test_unpack_only(self): test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir())) self._unpack_scheduler.add_task(test_fw) processed_container = self._tmp_queue.get(timeout=5) self.assertEqual(len(processed_container.files_included), 3, 'not all included files found') self.assertIn('faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', processed_container.files_included, 'certain file missing after unpacking')
def test_get_objects_by_uid_list(self): self.db_interface_backend.add_firmware(self.test_firmware) fo_list = self.db_interface.get_objects_by_uid_list([self.test_firmware.uid]) self.assertIsInstance(fo_list[0], Firmware, 'firmware has wrong type') self.assertEqual(fo_list[0].device_name, 'test_router', 'Device name in Firmware not correct') test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2')) self.db_interface_backend.add_file_object(test_file) fo_list = self.db_interface.get_objects_by_uid_list([test_file.uid]) self.assertIsInstance(fo_list[0], FileObject, 'file object has wrong type')
def test_skip_analysis_because_whitelist(self): self.sched.config.set('dummy_plugin_for_testing_only', 'mime_whitelist', 'foo, bar') test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1')) test_fw.scheduled_analysis = ['file_hashes'] test_fw.processed_analysis['file_type'] = {'mime': 'text/plain'} self.sched._start_or_skip_analysis('dummy_plugin_for_testing_only', test_fw) test_fw = self.tmp_queue.get(timeout=10) assert 'dummy_plugin_for_testing_only' in test_fw.processed_analysis assert 'skipped' in test_fw.processed_analysis['dummy_plugin_for_testing_only']
def test_read_list_from_config(monkeypatch, input_data, expected): monkeypatch.setattr( 'helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) test_config = load_config('test.cfg') test_config.add_section('test_section') test_config.set('test_section', 'test_option', input_data) result = read_list_from_config(test_config, 'test_section', 'test_option') assert result == expected
def test_process_object(self): test_file = FileObject( file_path=os.path.join(get_test_data_dir(), "yara_test_file")) test_file.processed_analysis.update({self.PLUGIN_NAME: []}) processed_file = self.analysis_plugin.process_object(test_file) results = processed_file.processed_analysis[self.PLUGIN_NAME] self.assertEqual(len(results), 2, "not all matches found") self.assertTrue('testRule' in results, "testRule match not found") self.assertEqual(results['summary'], ['testRule'])
def test_read_list_from_config__key_not_in_config(monkeypatch): monkeypatch.setattr( 'helperFunctions.config.get_config_dir', lambda: '{}/helperFunctions'.format(get_test_data_dir())) test_config = load_config('test.cfg') result = read_list_from_config(test_config, 'foo', 'bar') assert result == [] result = read_list_from_config(test_config, 'test', 'bar') assert result == []
def test_whole_run_analysis_selected(self): test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1')) test_fw.scheduled_analysis = ['dummy_plugin_for_testing_only'] self.sched.start_analysis_of_object(test_fw) for _ in range(3): # 3 plugins have to run test_fw = self.tmp_queue.get(timeout=10) self.assertEqual(len(test_fw.processed_analysis), 3, 'analysis not done') self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['1'], 'first result', 'result not correct') self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['summary'], ['first result', 'second result']) self.assertIn('file_hashes', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed') self.assertIn('file_type', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
def test_process_object_nothing_found(self): test_file = FileObject( file_path=os.path.join(get_test_data_dir(), 'zero_byte')) test_file.processed_analysis.update({self.PLUGIN_NAME: []}) processed_file = self.analysis_plugin.process_object(test_file) self.assertEqual( len(processed_file.processed_analysis[self.PLUGIN_NAME]), 1, 'result present but should not') self.assertEqual( processed_file.processed_analysis[self.PLUGIN_NAME]['summary'], [], 'summary not empty')
def test_get_object(self): fo = self.db_interface.get_object(self.test_firmware.uid) self.assertIsNone(fo, 'found something but there is nothing in the database') self.db_interface_backend.add_firmware(self.test_firmware) fo = self.db_interface.get_object(self.test_firmware.uid) self.assertIsInstance(fo, Firmware, 'firmware has wrong type') self.assertEqual(fo.device_name, 'test_router', 'Device name in Firmware not correct') test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2')) self.db_interface_backend.add_file_object(test_file) fo = self.db_interface.get_object(test_file.uid) self.assertIsInstance(fo, FileObject, 'file object has wrong type')
def test_new_yara_matching(self): with open(os.path.join(get_test_data_dir(), 'yara_matches'), 'r') as fd: match_file = fd.read() matches = self.analysis_plugin._parse_yara_output(match_file) self.assertIsInstance(matches, dict, 'matches should be dict') self.assertIn('PgpPublicKeyBlock', matches.keys(), 'Pgp block should have been matched') self.assertIn(0, matches['PgpPublicKeyBlock']['strings'][0], 'first block should start at 0x0')
def test_detect_type_of_file(self): test_file = FileObject( file_path='{}/container/test.zip'.format(get_test_data_dir())) test_file = self.analysis_plugin.process_object(test_file) assert test_file.processed_analysis[self.PLUGIN_NAME][ 'mime'] == 'application/zip', 'mime-type not detected correctly' assert test_file.processed_analysis[ self.PLUGIN_NAME]['full'].startswith( 'Zip archive data, at least'), 'full type not correct' assert test_file.processed_analysis[self.PLUGIN_NAME]['summary'] == [ 'application/zip' ]
def test_unpack_and_analyse(self): test_fw = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) self._unpack_scheduler.add_task(test_fw) for _ in range( 4 * 2 ): # container with 3 included files times 2 mandatory plugins run processed_container = self._tmp_queue.get(timeout=10) self.assertGreaterEqual(len(processed_container.processed_analysis), 3, 'at least one analysis not done')
def test_unpack_a_container_including_another_container(self): self._start_scheduler() test_fw = Firmware(file_path='{}/container/test_zip.tar.gz'.format(get_test_data_dir())) self.scheduler.add_task(test_fw) outer_container = self.tmp_queue.get(timeout=5) self.assertEqual(len(outer_container.files_included), 2, 'not all childs of root found') self.assertIn('ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227', outer_container.files_included, 'included container not extracted. Unpacker tar.gz modul broken?') included_files = [self.tmp_queue.get(timeout=5)] included_files.append(self.tmp_queue.get(timeout=5)) for item in included_files: if item.uid == 'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227': self.assertEqual(len(item.files_included), 1, 'number of files in included container not correct') else: self.assertEqual(item.uid, 'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', 'none container file not rescheduled')
def setUp(self): yara_binary_search.YaraBinarySearchScannerDbInterface.__bases__ = ( MockCommonDbInterface, ) yara_binary_search.ConnectTo.__enter__ = mock_connect_to_enter yara_binary_search.ConnectTo.__exit__ = lambda _, __, ___, ____: None self.yara_rule = b'rule test_rule {strings: $a = "test1234" condition: $a}' test_path = path.join(get_test_data_dir(), TEST_FILE_1) test_config = { 'data_storage': { 'firmware_file_storage_directory': test_path } } self.yara_binary_scanner = yara_binary_search.YaraBinarySearchScanner( test_config)
def test_run_analysis_with_tag(self): test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir())) test_fw.release_date = '2017-01-01' test_fw.scheduled_analysis = ['crypto_material'] self._unpack_scheduler.add_task(test_fw) assert self.analysis_finished_event.wait(timeout=20) processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material']) assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis' processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material']) assert processed_fw.analysis_tags, 'tags not propagated properly' assert processed_fw.analysis_tags['crypto_material']['private_key_inside']