def test_process_next_analysis_unknown_plugin(self): test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1')) test_fw.scheduled_analysis = ['unknown_plugin'] with mock_spy(self.sched, '_start_or_skip_analysis') as spy: self.sched.process_next_analysis(test_fw) assert not spy.was_called(), 'unknown plugin should simply be skipped'
def test_add_tag(input_data, expected_count): test_object = Firmware() for item in input_data: test_object.set_tag(item) for item in input_data: assert item in test_object.tags assert len(test_object.tags.keys()) == expected_count
def test_set_binary(): binary = get_binary_from_file('{}/get_files_test/testfile1'.format( get_test_data_dir())) md5 = 'e802ca22f6cd2d9357cf3da1d191879e' firmware = Firmware() firmware.set_binary(binary) assert firmware.md5 == md5
def test_create_firmware_from_file(self): test_object = Firmware() test_object.create_from_file('{}/test_data_file.bin'.format(get_test_data_dir())) self.assertEqual(test_object.device_name, None, 'correct device name') self.assertEqual(test_object.size, 19, 'correct size') self.assertEqual(test_object.binary, b'test string in file', 'correct binary data') self.assertEqual(test_object.sha256, '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8', 'correct sha256') self.assertEqual(test_object.file_name, 'test_data_file.bin', 'correct file name')
def test_create_firmware_from_file(): test_object = Firmware() test_object.create_from_file('{}/test_data_file.bin'.format(get_test_data_dir())) assert test_object.device_name is None assert test_object.size == 19 assert test_object.binary == b'test string in file' assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8' assert test_object.file_name == 'test_data_file.bin'
def test_add_firmware_to_current_analyses(self): self.scheduler.currently_running = {} fw = Firmware(binary=b'foo') fw.files_included = ['foo', 'bar'] self.scheduler._add_to_current_analyses(fw) assert fw.uid in self.scheduler.currently_running assert self.scheduler.currently_running[fw.uid]['file_list'] == ['foo', 'bar'] assert self.scheduler.currently_running[fw.uid]['analyzed_files_count'] == 0 assert self.scheduler.currently_running[fw.uid]['total_files_count'] == 2
def test_skip_analysis_because_whitelist(self): self.sched.config.set('dummy_plugin_for_testing_only', 'mime_whitelist', 'foo, bar') test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1')) test_fw.scheduled_analysis = ['file_hashes'] test_fw.processed_analysis['file_type'] = {'mime': 'text/plain'} self.sched._start_or_skip_analysis('dummy_plugin_for_testing_only', test_fw) test_fw = self.tmp_queue.get(timeout=10) assert 'dummy_plugin_for_testing_only' in test_fw.processed_analysis assert 'skipped' in test_fw.processed_analysis['dummy_plugin_for_testing_only']
def test_whole_run_analysis_selected(self): test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1')) test_fw.scheduled_analysis = ['dummy_plugin_for_testing_only'] self.sched.add_task(test_fw) test_fw = self.tmp_queue.get(timeout=10) self.assertEqual(len(test_fw.processed_analysis), 3, 'analysis not done') self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['1'], 'first result', 'result not correct') self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['summary'], ['first result', 'second result']) self.assertIn('file_hashes', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed') self.assertIn('file_type', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
def test_add_firmware_to_current_analyses(self): self.status.currently_running = {} fw = Firmware(binary=b'foo') fw.files_included = ['foo', 'bar'] self.status.add_to_current_analyses(fw) assert fw.uid in self.status.currently_running result = self.status.currently_running[fw.uid] assert result['files_to_unpack'] == ['foo', 'bar'] assert result['files_to_analyze'] == [fw.uid] assert result['unpacked_files_count'] == 1 assert result['analyzed_files_count'] == 0 assert result['total_files_count'] == 3
def test_get_hid(input_data, expected_output): test_fw = Firmware(binary=b'foo') test_fw.device_name = 'test_device' test_fw.vendor = 'foo' test_fw.version = '1.0' test_fw.set_part_name(input_data) assert test_fw.get_hid() == expected_output
def test_reschedule_failed_analysis_task(self): task = Firmware(binary='foo') error_message = 'There was an exception' task.analysis_exception = ('foo', error_message) task.scheduled_analysis = ['no_deps', 'bar'] task.processed_analysis['foo'] = {'error': 1} self._add_plugins() self.scheduler._reschedule_failed_analysis_task(task) assert 'foo' in task.processed_analysis assert task.processed_analysis['foo'] == {'failed': error_message} assert 'bar' not in task.scheduled_analysis assert 'bar' in task.processed_analysis assert task.processed_analysis['bar'] == {'failed': 'Analysis of dependency foo failed'} assert 'no_deps' in task.scheduled_analysis
def test_run_analysis_with_tag(self): test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir())) test_fw.release_date = '2017-01-01' test_fw.scheduled_analysis = ['crypto_material'] self._unpack_scheduler.add_task(test_fw) assert self.analysis_finished_event.wait(timeout=20) processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material']) assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis' processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material']) assert processed_fw.analysis_tags, 'tags not propagated properly' assert processed_fw.analysis_tags['crypto_material']['private_key_inside']
def test_unpack_a_container_including_another_container(self): self._start_scheduler() test_fw = Firmware(file_path='{}/container/test_zip.tar.gz'.format( get_test_data_dir())) self.sched.add_task(test_fw) outer_container = self.tmp_queue.get(timeout=5) self.assertEqual(len(outer_container.files_included), 2, 'not all childs of root found') self.assertIn( 'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227', outer_container.files_included, 'included container not extracted. Unpacker tar.gz modul broken?') included_files = [self.tmp_queue.get(timeout=5)] included_files.append(self.tmp_queue.get(timeout=5)) for item in included_files: if item.get_uid( ) == 'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227': self.assertEqual( len(item.files_included), 1, 'number of files in included container not correct') else: self.assertEqual( item.get_uid(), 'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', 'none container file not rescheduled')
def convert_analysis_task_to_fw_obj(analysis_task: dict, base_fw: Optional[Firmware] = None ) -> Firmware: ''' Convert an analysis task to a firmware object. :param analysis_task: The analysis task data. :param base_fw: The existing `Firmware` object in case of analysis update. :return: A `Firmware` object based on the analysis task data. ''' fw = base_fw or Firmware() fw.scheduled_analysis = analysis_task['requested_analysis_systems'] if 'binary' in analysis_task.keys(): fw.set_binary(analysis_task['binary']) fw.file_name = analysis_task['file_name'] else: if 'file_name' in analysis_task.keys(): fw.file_name = analysis_task['file_name'] fw.uid = analysis_task['uid'] fw.device_name = analysis_task['device_name'] fw.set_part_name(analysis_task['device_part']) fw.version = analysis_task['version'] fw.device_class = analysis_task['device_class'] fw.vendor = analysis_task['vendor'] fw.release_date = analysis_task['release_date'] for tag in _get_tag_list(analysis_task['tags']): fw.set_tag(tag) return fw
def test_unpack_and_analyse(self): test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir())) self._unpack_scheduler.add_task(test_fw) processed_container = self._tmp_queue.get(timeout=10) self.assertGreaterEqual(len(processed_container.processed_analysis), 3, 'at least one analysis not done')
def test_schedule_firmware_init_no_analysis_selected(self): self.sched.shutdown() self.sched.process_queue = Queue() test_fw = Firmware(binary=b'test') self.sched.add_task(test_fw) test_fw = self.sched.process_queue.get(timeout=5) self.assertEqual(len(test_fw.scheduled_analysis), len(MANDATORY_PLUGINS), 'Mandatory Plugins not selected') for item in MANDATORY_PLUGINS: self.assertIn(item, test_fw.scheduled_analysis)
def test_unpack_only(self): test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir())) self._unpack_scheduler.add_task(test_fw) processed_container = self._tmp_queue.get(timeout=5) self.assertEqual(len(processed_container.files_included), 3, 'not all included files found') self.assertIn('faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', processed_container.files_included, 'certain file missing after unpacking')
def _init_current_analysis(fw_object: Firmware): return { 'files_to_unpack': list(fw_object.files_included), 'files_to_analyze': [fw_object.uid], 'start_time': time(), 'unpacked_files_count': 1, 'analyzed_files_count': 0, 'total_files_count': 1 + len(fw_object.files_included), 'hid': fw_object.get_hid(), }
def test_unpack_analyse_and_compare(self): test_fw_1 = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) test_fw_1.release_date = '2017-01-01' test_fw_2 = Firmware( file_path='{}/container/test.7z'.format(get_test_data_dir())) test_fw_2.release_date = '2017-01-01' self._unpack_scheduler.add_task(test_fw_1) self._unpack_scheduler.add_task(test_fw_2) self.analysis_finished_event.wait(timeout=10) compare_id = unify_string_list(';'.join( [fw.uid for fw in [test_fw_1, test_fw_2]])) self.assertIsNone( self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error') self.compare_finished_event.wait(timeout=10) with ConnectTo(CompareDbInterface, self._config) as sc: result = sc.get_compare_result(compare_id) self.assertFalse(isinstance(result, str), 'compare result should exist') self.assertEqual(result['plugins']['Software'], self._expected_result()['Software']) self.assertCountEqual( result['plugins']['File_Coverage']['exclusive_files'], self._expected_result()['File_Coverage']['exclusive_files'])
def test_unpack_analyse_and_compare(self): test_fw_1 = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) test_fw_1.release_date = '2017-01-01' test_fw_2 = Firmware( file_path='{}/regression_one'.format(get_test_data_dir())) test_fw_2.release_date = '2017-01-01' self._unpack_scheduler.add_task(test_fw_1) self._unpack_scheduler.add_task(test_fw_2) self.analysis_finished_event.wait(timeout=20) compare_id = normalize_compare_id(';'.join( [fw.uid for fw in [test_fw_1, test_fw_2]])) self.assertIsNone( self._compare_scheduler.add_task((compare_id, False)), 'adding compare task creates error') self.compare_finished_event.wait(timeout=10) with ConnectTo(CompareDbInterface, self._config) as sc: result = sc.get_compare_result(compare_id) self.assertEqual(result['plugins']['Software'], self._expected_result()['Software']) self.assertCountEqual( result['plugins']['File_Coverage']['files_in_common'], self._expected_result()['File_Coverage']['files_in_common'])
def test_unpack_and_analyse(self): test_fw = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) self._unpack_scheduler.add_task(test_fw) for _ in range( 12 ): # magic to make tests pass. after switch of post analysis, each analysis gets the post call processed_container = self._tmp_queue.get(timeout=10) self.assertGreaterEqual(len(processed_container.processed_analysis), 3, 'at least one analysis not done')
def test_unpack_and_analyse(self): test_fw = Firmware( file_path='{}/container/test.zip'.format(get_test_data_dir())) self._unpack_scheduler.add_task(test_fw) for _ in range( 4 * 2 ): # container with 3 included files times 2 mandatory plugins run processed_container = self._tmp_queue.get(timeout=10) self.assertGreaterEqual(len(processed_container.processed_analysis), 3, 'at least one analysis not done')
def _convert_to_firmware(self, entry: dict, analysis_filter: List[str] = None) -> Firmware: firmware = Firmware() firmware.uid = entry['_id'] firmware.size = entry['size'] firmware.file_name = entry['file_name'] firmware.device_name = entry['device_name'] firmware.device_class = entry['device_class'] firmware.release_date = convert_time_to_str(entry['release_date']) firmware.vendor = entry['vendor'] firmware.version = entry['version'] firmware.processed_analysis = self.retrieve_analysis(entry['processed_analysis'], analysis_filter=analysis_filter) firmware.files_included = set(entry['files_included']) firmware.virtual_file_path = entry['virtual_file_path'] firmware.tags = entry['tags'] if 'tags' in entry else dict() firmware.analysis_tags = self._collect_analysis_tags_from_children(firmware.uid) try: # for backwards compatibility firmware.set_part_name(entry['device_part']) except KeyError: firmware.set_part_name('complete') if 'comments' in entry: # for backwards compatibility firmware.comments = entry['comments'] return firmware
def test_get_hid(input_data, expected_output): test_fw = Firmware(binary=b'foo') test_fw.set_device_name('test_device') test_fw.set_vendor('foo') test_fw.set_firmware_version('1.0') test_fw.set_part_name(input_data) assert test_fw.get_hid() == expected_output
def test_repr_and_str(): test_fw = Firmware(scheduled_analysis=['test']) assert 'None None v. None' in test_fw.__str__() assert 'test' in test_fw.__str__() assert test_fw.__str__() == test_fw.__repr__()
def _convert_to_firmware(self, entry, analysis_filter=None): firmware = Firmware() firmware.uid = entry['_id'] firmware.size = entry['size'] firmware.set_name(entry['file_name']) firmware.set_device_name(entry['device_name']) firmware.set_device_class(entry['device_class']) firmware.set_release_date(convert_time_to_str(entry['release_date'])) firmware.set_vendor(entry['vendor']) firmware.set_firmware_version(entry['version']) firmware.processed_analysis = self.retrieve_analysis(entry['processed_analysis'], analysis_filter=analysis_filter) firmware.files_included = set(entry['files_included']) firmware.virtual_file_path = entry['virtual_file_path'] firmware.tags = entry['tags'] if 'tags' in entry else dict() firmware.analysis_tags = entry['analysis_tags'] if 'analysis_tags' in entry else dict() if 'comments' in entry: # for backwards compatibility firmware.comments = entry['comments'] return firmware
def test_create_firmware_container_raw(): test_object = Firmware() assert test_object.size is None assert test_object.binary is None
def test_remove_tag(tag_set, remove_items, expected_count): test_fw = Firmware() test_fw.tags = tag_set for item in remove_items: test_fw.remove_tag(item) assert len(test_fw.tags.keys()) == expected_count
def test_set_part_name(input_data, expected_output): test_object = Firmware() test_object.set_part_name(input_data) assert test_object.part == expected_output
def create_test_firmware(device_class='Router', device_name='test_router', vendor='test_vendor', bin_path='container/test.zip', all_files_included_set=False, version='0.1'): fw = Firmware(file_path=os.path.join(get_test_data_dir(), bin_path)) fw.set_device_class(device_class) fw.set_device_name(device_name) fw.set_vendor(vendor) fw.set_release_date('1970-01-01') fw.version = version processed_analysis = { 'dummy': { 'summary': ['sum a', 'fw exclusive sum a'], 'content': 'abcd' }, 'unpacker': { 'plugin_used': 'used_unpack_plugin' }, 'file_type': { 'mime': 'test_type', 'full': 'Not a PE file', 'summary': ['a summary'] } } fw.processed_analysis.update(processed_analysis) if all_files_included_set: fw.list_of_all_included_files = list(fw.files_included) fw.list_of_all_included_files.append(fw.uid) return fw