def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
def test_execute_yara_search_for_single_file(self):
test_rule_path = path.join(get_test_data_dir(),
'yara_binary_search_test_rule')
result = self.yara_binary_scanner._execute_yara_search(
test_rule_path,
target_path=path.join(get_test_data_dir(), TEST_FILE_1,
TEST_FILE_1))
self.assertTrue('test_rule' in result)
def test_get_temp_dir_path(caplog):
assert get_temp_dir_path() == '/tmp'
assert get_temp_dir_path(MockConfig(
get_test_data_dir())) == get_test_data_dir()
not_a_dir = str(Path(get_test_data_dir()) / '__init__.py')
with caplog.at_level(logging.WARNING):
assert get_temp_dir_path(MockConfig(not_a_dir)) == '/tmp'
assert 'TempDir path does not exist and could not be created' in caplog.messages[
0]
def test_file_is_zero(self):
self.assertTrue(
file_is_empty('{}/zero_byte'.format(get_test_data_dir())),
'file is empty but stated differently')
self.assertFalse(
file_is_empty('{}/get_files_test/testfile1'.format(
get_test_data_dir())), 'file not empty but stated differently')
self.assertFalse(
file_is_empty(os.path.join(get_test_data_dir(), 'broken_link')),
'Broken link is not empty')
def test_file_is_locked(self):
assert not self.unpacker.db_interface.check_unpacking_lock(
self.test_fo.uid)
file_paths = [
'{}/get_files_test/testfile1'.format(get_test_data_dir())
]
self.unpacker.generate_and_store_file_objects(file_paths,
get_test_data_dir(),
self.test_fo)
assert self.unpacker.db_interface.check_unpacking_lock(
self.test_fo.uid)
def test_dont_store_zero_file(self):
file_pathes = [
Path(get_test_data_dir(), 'files', 'zero_byte'),
Path(get_test_data_dir(), 'files', 'get_files_test', 'testfile1')
]
file_objects = self.unpacker.generate_and_store_file_objects(
file_pathes, get_test_data_dir(), self.test_fo)
file_objects = make_list_from_dict(file_objects)
self.assertEqual(len(file_objects), 1, 'number of objects not correct')
self.assertEqual(file_objects[0].file_name, 'testfile1',
'wrong object created')
parent_uid = self.test_fo.uid
self.assertIn('|{}|/get_files_test/testfile1'.format(parent_uid),
file_objects[0].virtual_file_path[self.test_fo.uid])
class argument_mock():
config_file = get_test_data_dir() + '/load_cfg_test'
log_file = '/log/file/path'
log_level = 'DEBUG'
silent = False
debug = False
def test_process_next_analysis_unknown_plugin(self):
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['unknown_plugin']
with mock_spy(self.sched, '_start_or_skip_analysis') as spy:
self.sched.process_next_analysis(test_fw)
assert not spy.was_called(), 'unknown plugin should simply be skipped'
def get_file_name_and_path(self, uid):
if uid != 'foo':
return {
'file_path':
os.path.join(get_test_data_dir(), 'container/test.zip')
}
return None
def test_set_binary():
binary = get_binary_from_file('{}/get_files_test/testfile1'.format(
get_test_data_dir()))
md5 = 'e802ca22f6cd2d9357cf3da1d191879e'
firmware = Firmware()
firmware.set_binary(binary)
assert firmware.md5 == md5
class TestUnpackerCoreMain(TestUnpackerBase):
test_file_path = str(Path(get_test_data_dir()) / 'container/test.zip')
def main_unpack_check(self, test_object, number_unpacked_files,
first_unpacker):
extracted_files = self.unpacker.unpack(test_object)
assert len(test_object.files_included
) == number_unpacked_files, 'not all files added to parent'
assert len(
extracted_files) == number_unpacked_files, 'not all files found'
assert test_object.processed_analysis['unpacker'][
'plugin_used'] == first_unpacker, 'Wrong plugin in Meta'
assert test_object.processed_analysis['unpacker'][
'number_of_unpacked_files'] == number_unpacked_files, 'Number of unpacked files wrong in Meta'
self.check_depths_of_children(test_object, extracted_files)
@staticmethod
def check_depths_of_children(parent, extracted_files):
for item in extracted_files:
assert item.depth == parent.depth + 1, 'depth of child not correct'
def test_main_unpack_function(self):
test_file = FileObject(file_path=self.test_file_path)
self.main_unpack_check(test_file, 3, '7z')
def test_unpacking_depth_reached(self):
test_file = FileObject(file_path=self.test_file_path)
test_file.depth = 10
self.unpacker.unpack(test_file)
assert 'unpacker' in test_file.processed_analysis
assert 'maximum unpacking depth was reached' in test_file.processed_analysis[
'unpacker']['info']
def test_imphash():
fo = create_test_file_object(
bin_path=str(Path(get_test_data_dir(), 'test_executable')))
fo.processed_analysis = {'file_type': {'mime': 'application/x-executable'}}
imphash = get_imphash(fo)
assert isinstance(imphash, str), 'imphash should be a string'
assert len(imphash) == 32, 'imphash does not seem to be an md5'
def test_process_object(self):
test_file = FileObject(file_path=os.path.join(get_test_data_dir(), 'yara_test_file'))
test_file.processed_analysis.update({self.PLUGIN_NAME: []})
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
assert len(results) == 2, 'not all matches found'
assert 'testRule' in results, 'testRule match not found'
assert results['summary'] == ['testRule']
def test_create_firmware_from_file():
test_object = Firmware(
file_path='{}/test_data_file.bin'.format(get_test_data_dir()))
assert test_object.device_name is None
assert test_object.size == 19
assert test_object.binary == b'test string in file'
assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8'
assert test_object.file_name == 'test_data_file.bin'
def test_get_file_from_binary(self):
file_path = '{}/test_data_file.bin'.format(get_test_data_dir())
test_object = FileObject(file_path=file_path)
assert test_object.size == 19, 'correct size'
assert test_object.binary == b'test string in file', 'correct binary data'
assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8', 'correct sha256'
assert test_object.file_name == 'test_data_file.bin', 'correct file name'
assert test_object.file_path == file_path, 'correct file path'
def test_unpack_only(self):
test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
processed_container = self._tmp_queue.get(timeout=5)
self.assertEqual(len(processed_container.files_included), 3, 'not all included files found')
self.assertIn('faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', processed_container.files_included, 'certain file missing after unpacking')
def test_get_objects_by_uid_list(self):
self.db_interface_backend.add_firmware(self.test_firmware)
fo_list = self.db_interface.get_objects_by_uid_list([self.test_firmware.uid])
self.assertIsInstance(fo_list[0], Firmware, 'firmware has wrong type')
self.assertEqual(fo_list[0].device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo_list = self.db_interface.get_objects_by_uid_list([test_file.uid])
self.assertIsInstance(fo_list[0], FileObject, 'file object has wrong type')
def test_skip_analysis_because_whitelist(self):
self.sched.config.set('dummy_plugin_for_testing_only', 'mime_whitelist', 'foo, bar')
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['file_hashes']
test_fw.processed_analysis['file_type'] = {'mime': 'text/plain'}
self.sched._start_or_skip_analysis('dummy_plugin_for_testing_only', test_fw)
test_fw = self.tmp_queue.get(timeout=10)
assert 'dummy_plugin_for_testing_only' in test_fw.processed_analysis
assert 'skipped' in test_fw.processed_analysis['dummy_plugin_for_testing_only']
def test_read_list_from_config(monkeypatch, input_data, expected):
monkeypatch.setattr(
'helperFunctions.config.get_config_dir',
lambda: '{}/helperFunctions'.format(get_test_data_dir()))
test_config = load_config('test.cfg')
test_config.add_section('test_section')
test_config.set('test_section', 'test_option', input_data)
result = read_list_from_config(test_config, 'test_section', 'test_option')
assert result == expected
def test_process_object(self):
test_file = FileObject(
file_path=os.path.join(get_test_data_dir(), "yara_test_file"))
test_file.processed_analysis.update({self.PLUGIN_NAME: []})
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
self.assertEqual(len(results), 2, "not all matches found")
self.assertTrue('testRule' in results, "testRule match not found")
self.assertEqual(results['summary'], ['testRule'])
def test_read_list_from_config__key_not_in_config(monkeypatch):
monkeypatch.setattr(
'helperFunctions.config.get_config_dir',
lambda: '{}/helperFunctions'.format(get_test_data_dir()))
test_config = load_config('test.cfg')
result = read_list_from_config(test_config, 'foo', 'bar')
assert result == []
result = read_list_from_config(test_config, 'test', 'bar')
assert result == []
def test_whole_run_analysis_selected(self):
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['dummy_plugin_for_testing_only']
self.sched.start_analysis_of_object(test_fw)
for _ in range(3): # 3 plugins have to run
test_fw = self.tmp_queue.get(timeout=10)
self.assertEqual(len(test_fw.processed_analysis), 3, 'analysis not done')
self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['1'], 'first result', 'result not correct')
self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['summary'], ['first result', 'second result'])
self.assertIn('file_hashes', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
self.assertIn('file_type', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
def test_process_object_nothing_found(self):
test_file = FileObject(
file_path=os.path.join(get_test_data_dir(), 'zero_byte'))
test_file.processed_analysis.update({self.PLUGIN_NAME: []})
processed_file = self.analysis_plugin.process_object(test_file)
self.assertEqual(
len(processed_file.processed_analysis[self.PLUGIN_NAME]), 1,
'result present but should not')
self.assertEqual(
processed_file.processed_analysis[self.PLUGIN_NAME]['summary'], [],
'summary not empty')
def test_get_object(self):
fo = self.db_interface.get_object(self.test_firmware.uid)
self.assertIsNone(fo, 'found something but there is nothing in the database')
self.db_interface_backend.add_firmware(self.test_firmware)
fo = self.db_interface.get_object(self.test_firmware.uid)
self.assertIsInstance(fo, Firmware, 'firmware has wrong type')
self.assertEqual(fo.device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo = self.db_interface.get_object(test_file.uid)
self.assertIsInstance(fo, FileObject, 'file object has wrong type')
def test_new_yara_matching(self):
with open(os.path.join(get_test_data_dir(), 'yara_matches'),
'r') as fd:
match_file = fd.read()
matches = self.analysis_plugin._parse_yara_output(match_file)
self.assertIsInstance(matches, dict, 'matches should be dict')
self.assertIn('PgpPublicKeyBlock', matches.keys(),
'Pgp block should have been matched')
self.assertIn(0, matches['PgpPublicKeyBlock']['strings'][0],
'first block should start at 0x0')
def test_detect_type_of_file(self):
test_file = FileObject(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_file = self.analysis_plugin.process_object(test_file)
assert test_file.processed_analysis[self.PLUGIN_NAME][
'mime'] == 'application/zip', 'mime-type not detected correctly'
assert test_file.processed_analysis[
self.PLUGIN_NAME]['full'].startswith(
'Zip archive data, at least'), 'full type not correct'
assert test_file.processed_analysis[self.PLUGIN_NAME]['summary'] == [
'application/zip'
]
def test_unpack_and_analyse(self):
test_fw = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
for _ in range(
4 * 2
): # container with 3 included files times 2 mandatory plugins run
processed_container = self._tmp_queue.get(timeout=10)
self.assertGreaterEqual(len(processed_container.processed_analysis), 3,
'at least one analysis not done')
def test_unpack_a_container_including_another_container(self):
self._start_scheduler()
test_fw = Firmware(file_path='{}/container/test_zip.tar.gz'.format(get_test_data_dir()))
self.scheduler.add_task(test_fw)
outer_container = self.tmp_queue.get(timeout=5)
self.assertEqual(len(outer_container.files_included), 2, 'not all childs of root found')
self.assertIn('ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227', outer_container.files_included, 'included container not extracted. Unpacker tar.gz modul broken?')
included_files = [self.tmp_queue.get(timeout=5)]
included_files.append(self.tmp_queue.get(timeout=5))
for item in included_files:
if item.uid == 'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227':
self.assertEqual(len(item.files_included), 1, 'number of files in included container not correct')
else:
self.assertEqual(item.uid, 'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', 'none container file not rescheduled')
def setUp(self):
yara_binary_search.YaraBinarySearchScannerDbInterface.__bases__ = (
MockCommonDbInterface, )
yara_binary_search.ConnectTo.__enter__ = mock_connect_to_enter
yara_binary_search.ConnectTo.__exit__ = lambda _, __, ___, ____: None
self.yara_rule = b'rule test_rule {strings: $a = "test1234" condition: $a}'
test_path = path.join(get_test_data_dir(), TEST_FILE_1)
test_config = {
'data_storage': {
'firmware_file_storage_directory': test_path
}
}
self.yara_binary_scanner = yara_binary_search.YaraBinarySearchScanner(
test_config)
def test_run_analysis_with_tag(self):
test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir()))
test_fw.release_date = '2017-01-01'
test_fw.scheduled_analysis = ['crypto_material']
self._unpack_scheduler.add_task(test_fw)
assert self.analysis_finished_event.wait(timeout=20)
processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material'])
assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis'
processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material'])
assert processed_fw.analysis_tags, 'tags not propagated properly'
assert processed_fw.analysis_tags['crypto_material']['private_key_inside']
