def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/container/test.7z'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=10)
compare_id = unify_string_list(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertFalse(isinstance(result, str),
'compare result should exist')
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['exclusive_files'],
self._expected_result()['File_Coverage']['exclusive_files'])
def test_execute_yara_search_for_single_file(self):
test_rule_path = path.join(fileSystem.get_test_data_dir(), 'yara_binary_search_test_rule')
result = self.yara_binary_scanner._execute_yara_search(
test_rule_path,
target_path=path.join(fileSystem.get_test_data_dir(), TEST_FILE_1, TEST_FILE_1)
)
self.assertTrue(b'test_rule' in result)
def test_generate_and_store_file_objects_zero_file(self):
file_pathes = ['{}/zero_byte'.format(get_test_data_dir()), '{}/get_files_test/testfile1'.format(get_test_data_dir())]
file_objects = self.unpacker.generate_and_store_file_objects(file_pathes, get_test_data_dir(), self.test_fo)
file_objects = make_list_from_dict(file_objects)
self.assertEqual(len(file_objects), 1, 'number of objects not correct')
self.assertEqual(file_objects[0].file_name, 'testfile1', 'wrong object created')
parentID = self.test_fo.get_uid()
self.assertIn('|{}|/get_files_test/testfile1'.format(parentID), file_objects[0].virtual_file_path[self.test_fo.get_uid()])
def test_file_is_zero(self):
self.assertTrue(
file_is_empty('{}/zero_byte'.format(get_test_data_dir())),
'file is empty but stated differently')
self.assertFalse(
file_is_empty('{}/get_files_test/testfile1'.format(
get_test_data_dir())), 'file not empty but stated differently')
self.assertFalse(
file_is_empty(os.path.join(get_test_data_dir(), 'broken_link')),
'Broken link is not empty')
def test_file_is_locked(self):
assert not self.unpacker.db_interface.check_unpacking_lock(
self.test_fo.uid)
file_paths = [
'{}/get_files_test/testfile1'.format(get_test_data_dir())
]
self.unpacker.generate_and_store_file_objects(file_paths,
get_test_data_dir(),
self.test_fo)
assert self.unpacker.db_interface.check_unpacking_lock(
self.test_fo.uid)
def test_dont_store_zero_file(self):
file_pathes = [
Path(get_test_data_dir(), 'files', 'zero_byte'),
Path(get_test_data_dir(), 'files', 'get_files_test', 'testfile1')
]
file_objects = self.unpacker.generate_and_store_file_objects(
file_pathes, get_test_data_dir(), self.test_fo)
file_objects = make_list_from_dict(file_objects)
self.assertEqual(len(file_objects), 1, 'number of objects not correct')
self.assertEqual(file_objects[0].file_name, 'testfile1',
'wrong object created')
parent_uid = self.test_fo.uid
self.assertIn('|{}|/get_files_test/testfile1'.format(parent_uid),
file_objects[0].virtual_file_path[self.test_fo.uid])
def test_entropy_graph(self):
test_file = FileObject(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
self.assertGreater(len(results['entropy_analysis_graph']), 0,
'no binwalk entropy graph found')
def test_extraction_bad_file(self):
file_path = str(Path(get_test_data_dir(), 'test_data_file.bin'))
with TemporaryDirectory() as tmp_dir:
meta_data = unpack_function(file_path, tmp_dir)
assert 'Invalid' in meta_data['output']
def get_file_name_and_path(self, uid):
if uid != 'foo':
return {
'file_path':
os.path.join(get_test_data_dir(), 'container/test.zip')
}
return None
def test_set_binary():
binary = get_binary_from_file('{}/get_files_test/testfile1'.format(
get_test_data_dir()))
md5 = 'e802ca22f6cd2d9357cf3da1d191879e'
firmware = Firmware()
firmware.set_binary(binary)
assert firmware.md5 == md5
def test_unpack_a_container_including_another_container(self):
self._start_scheduler()
test_fw = Firmware(file_path='{}/container/test_zip.tar.gz'.format(
get_test_data_dir()))
self.sched.add_task(test_fw)
outer_container = self.tmp_queue.get(timeout=5)
self.assertEqual(len(outer_container.files_included), 2,
'not all childs of root found')
self.assertIn(
'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227',
outer_container.files_included,
'included container not extracted. Unpacker tar.gz modul broken?')
included_files = [self.tmp_queue.get(timeout=5)]
included_files.append(self.tmp_queue.get(timeout=5))
for item in included_files:
if item.get_uid(
) == 'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227':
self.assertEqual(
len(item.files_included), 1,
'number of files in included container not correct')
else:
self.assertEqual(
item.get_uid(),
'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28',
'none container file not rescheduled')
class argument_mock():
config_file = get_test_data_dir() + '/load_cfg_test'
log_file = '/log/file/path'
log_level = 'DEBUG'
silent = False
debug = False
def test_process_next_analysis_unknown_plugin(self):
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['unknown_plugin']
with mock_spy(self.sched, '_start_or_skip_analysis') as spy:
self.sched.process_next_analysis(test_fw)
assert not spy.was_called(), 'unknown plugin should simply be skipped'
def create_test_firmware(device_class='Router',
device_name='test_router',
vendor='test_vendor',
bin_path='container/test.zip',
all_files_included_set=False,
version='0.1'):
fw = Firmware(file_path=os.path.join(get_test_data_dir(), bin_path))
fw.set_device_class(device_class)
fw.set_device_name(device_name)
fw.set_vendor(vendor)
fw.set_release_date('1970-01-01')
fw.version = version
processed_analysis = {
'dummy': {
'summary': ['sum a', 'fw exclusive sum a'],
'content': 'abcd'
},
'unpacker': {
'plugin_used': 'used_unpack_plugin'
},
'file_type': {
'mime': 'test_type',
'full': 'Not a PE file'
}
}
fw.processed_analysis.update(processed_analysis)
if all_files_included_set:
fw.list_of_all_included_files = list(fw.files_included)
fw.list_of_all_included_files.append(fw.get_uid())
return fw
def test_tar_repack(self):
file_path = os.path.join(get_test_data_dir(), 'container/test.zip')
result = self.repack_service.tar_repack(file_path)
file_type = magic.from_buffer(result, mime=True)
self.assertTrue(
file_type == 'application/x-gzip'
or file_type == 'application/gzip', 'Result is not an tar.gz file')
def create_test_file_object(bin_path='get_files_test/testfile1'):
fo = FileObject(file_path=os.path.join(get_test_data_dir(), bin_path))
processed_analysis = {'dummy': {'summary': [
'sum a', 'file exclusive sum b'], 'content': 'file abcd'}, 'file_type': {'full': 'Not a PE file'}}
fo.processed_analysis.update(processed_analysis)
fo.virtual_file_path = fo.get_virtual_file_paths()
return fo
def _unpack_fallback_check(self, fallback_mime, fallback_plugin_name):
broken_zip = FileObject(file_path=os.path.join(get_test_data_dir(), 'container/broken.zip'))
self.unpacker.unpack(broken_zip)
self.assertEqual(broken_zip.processed_analysis['unpacker']['0_ERROR_7z'][0:6], '\n7-Zip')
self.assertEqual(broken_zip.processed_analysis['unpacker']['0_FALLBACK_7z'], '7z (failed) -> {} (fallback)'.format(fallback_mime))
self.assertEqual(broken_zip.processed_analysis['unpacker']['plugin_used'], fallback_plugin_name)
return broken_zip
def test_new_yara_matching(self):
with open(os.path.join(get_test_data_dir(), 'yara_matches'), 'r') as fd:
match_file = fd.read()
matches = self.analysis_plugin._parse_yara_output(match_file)
self.assertIsInstance(matches, dict, 'matches should be dict')
self.assertIn('PgpPublicKeyBlock', matches.keys(), 'Pgp block should have been matched')
self.assertIn(0, matches['PgpPublicKeyBlock']['strings'][0], 'first block should start at 0x0')
def test_process_object(self):
test_file = FileObject(file_path=os.path.join(get_test_data_dir(), "yara_test_file"))
test_file.processed_analysis.update({self.PLUGIN_NAME: []})
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
self.assertEqual(len(results), 2, "not all matches found")
self.assertTrue('testRule' in results, "testRule match not found")
self.assertEqual(results['summary'], ['testRule'])
def _set_config(cls):
super()._set_config()
cls.config.set('ExpertSettings', 'authentication', 'true')
cls.config.set('data_storage', 'user_database', ''.join(['sqlite:///', get_test_data_dir(), '/user_test.db']))
cls.guest = MockUser(name='t_guest', password='******', key='1okMSKUKlYxSvPn0sgfHM0SWd9zqNChyj5fbcIJgfKM=')
cls.guest_analyst = MockUser(name='t_guest_analyst', password='******', key='mDsgjAM2iE543PySnTpPZr0u8KeGTPGzPjKJVO4I4Ww=')
cls.superuser = MockUser(name='t_superuser', password='******', key='k2GKnNaA5UlENStVI4AEJKQ7BP9ZqO+21Cx746BjJDo=')
def setUp(self):
yara_binary_search.YaraBinarySearchScannerDbInterface.__bases__ = (MockCommonDbInterface,)
yara_binary_search.ConnectTo.__enter__ = mock_connect_to_enter
yara_binary_search.ConnectTo.__exit__ = lambda _, __, ___, ____: None
self.yara_rule = b'rule test_rule {strings: $a = "test1234" condition: $a}'
test_path = path.join(fileSystem.get_test_data_dir(), TEST_FILE_1)
test_config = {'data_storage': {'firmware_file_storage_directory': test_path}}
self.yara_binary_scanner = yara_binary_search.YaraBinarySearchScanner(test_config)
def test_unpack_and_analyse(self):
test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
processed_container = self._tmp_queue.get(timeout=10)
self.assertGreaterEqual(len(processed_container.processed_analysis), 3, 'at least one analysis not done')
def test_create_firmware_from_file(self):
test_object = Firmware()
test_object.create_from_file('{}/test_data_file.bin'.format(get_test_data_dir()))
self.assertEqual(test_object.device_name, None, 'correct device name')
self.assertEqual(test_object.size, 19, 'correct size')
self.assertEqual(test_object.binary, b'test string in file', 'correct binary data')
self.assertEqual(test_object.sha256, '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8', 'correct sha256')
self.assertEqual(test_object.file_name, 'test_data_file.bin', 'correct file name')
def test_create_firmware_from_file():
test_object = Firmware()
test_object.create_from_file('{}/test_data_file.bin'.format(get_test_data_dir()))
assert test_object.device_name is None
assert test_object.size == 19
assert test_object.binary == b'test string in file'
assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8'
assert test_object.file_name == 'test_data_file.bin'
def test_get_objects_by_uid_list(self):
self.db_interface_backend.add_firmware(self.test_firmware)
fo_list = self.db_interface.get_objects_by_uid_list([self.test_firmware.get_uid()])
self.assertIsInstance(fo_list[0], Firmware, 'firmware has wrong type')
self.assertEqual(fo_list[0].device_name, 'test_router', 'Device name in Firmware not correct')
test_file = FileObject(file_path=path.join(get_test_data_dir(), 'get_files_test/testfile2'))
self.db_interface_backend.add_file_object(test_file)
fo_list = self.db_interface.get_objects_by_uid_list([test_file.get_uid()])
self.assertIsInstance(fo_list[0], FileObject, 'file object has wrong type')
def test_skip_analysis_because_whitelist(self):
self.sched.config.set('dummy_plugin_for_testing_only', 'mime_whitelist', 'foo, bar')
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['file_hashes']
test_fw.processed_analysis['file_type'] = {'mime': 'text/plain'}
self.sched._start_or_skip_analysis('dummy_plugin_for_testing_only', test_fw)
test_fw = self.tmp_queue.get(timeout=10)
assert 'dummy_plugin_for_testing_only' in test_fw.processed_analysis
assert 'skipped' in test_fw.processed_analysis['dummy_plugin_for_testing_only']
def test_signature_analysis(self):
test_file = FileObject(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
self.assertGreater(len(results['signature_analysis']), 0,
'no binwalk signature analysis found')
self.assertTrue('DECIMAL' in results['signature_analysis'],
'no valid binwalk signature analysis')
def test_get_file_from_binary(self):
file_path = '{}/test_data_file.bin'.format(get_test_data_dir())
test_object = FileObject()
test_object.create_from_file(file_path)
self.assertEqual(test_object.size, 19, 'correct size')
self.assertEqual(test_object.binary, b'test string in file', 'correct binary data')
self.assertEqual(test_object.sha256, '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8', 'correct sha256')
self.assertEqual(test_object.file_name, 'test_data_file.bin', 'correct file name')
self.assertEqual(test_object.file_path, file_path, 'correct file path')
def test_whole_run_analysis_selected(self):
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['dummy_plugin_for_testing_only']
self.sched.add_task(test_fw)
test_fw = self.tmp_queue.get(timeout=10)
self.assertEqual(len(test_fw.processed_analysis), 3, 'analysis not done')
self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['1'], 'first result', 'result not correct')
self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['summary'], ['first result', 'second result'])
self.assertIn('file_hashes', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
self.assertIn('file_type', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')