def test_add_file_object_to_file_object():
root = create_test_firmware()
child_fo = create_test_file_object()
root.add_included_file(child_fo)
grandchild_fo = create_test_file_object(
bin_path='get_files_test/testfile2')
child_fo.add_included_file(grandchild_fo)
assert grandchild_fo.virtual_file_path[root.uid][0] == join_virtual_path(
root.uid, child_fo.uid, grandchild_fo.file_path)
def test_find_failed_analyses_with_multiple_files(self):
test_fo_1 = create_test_file_object()
test_fo_1.processed_analysis.update({'foo': {'failed': 'some reason'}})
test_fo_2 = create_test_file_object(bin_path='container/test.7z')
test_fo_2.processed_analysis.update({'foo': {'failed': 'no reason'}})
assert test_fo_1.uid != test_fo_2.uid, 'files should not be the same'
self.db_backend_interface.add_file_object(test_fo_1)
self.db_backend_interface.add_file_object(test_fo_2)
failed_analyses = self.db_frontend_interface.find_failed_analyses()
assert failed_analyses, 'should not be empty'
assert list(failed_analyses) == ['foo']
assert len(failed_analyses['foo']) == 2
assert test_fo_1.uid in failed_analyses['foo'] and test_fo_2.uid in failed_analyses['foo']
def test_generate_file_tree_level(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
child_fo.processed_analysis['file_type'] = {'mime': 'sometype'}
uid = parent_fw.uid
child_fo.virtual_file_path = {
uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]
}
parent_fw.files_included = {child_fo.uid}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
for node in self.db_frontend_interface.generate_file_tree_level(
uid, uid):
assert isinstance(node, FileTreeNode)
assert node.name == parent_fw.file_name
assert node.has_children
for node in self.db_frontend_interface.generate_file_tree_level(
child_fo.uid, uid):
assert isinstance(node, FileTreeNode)
assert node.name == 'folder'
assert node.has_children
virtual_grand_child = node.get_list_of_child_nodes()[0]
assert virtual_grand_child.type == 'sometype'
assert not virtual_grand_child.has_children
assert virtual_grand_child.name == child_fo.file_name
def test_get_results_from_parent_fos__multiple_vfps_in_one_fw(self):
fw = create_test_firmware()
fo = create_test_file_object()
file_names = ['file_a', 'file_b', 'file_c']
fw.processed_analysis[AnalysisPlugin.NAME] = {
'files': {b64_encode(f): {
'result': 'value'
}
for f in file_names}
}
vfp = fo.virtual_file_path['some_uid'] = []
for f in file_names:
vfp.append('some_uid|{}|/{}'.format(fw.uid, f))
results = {}
routes.FsMetadataRoutesDbInterface.get_results_from_parent_fos(
fw, fo, results)
assert results is not None
assert results != {}, 'result should not be empty'
assert len(results) == 3, 'wrong number of results'
assert all(f in results
for f in file_names), 'files missing from result'
assert 'result' in results[file_names[0]], 'analysis result is missing'
assert results[file_names[0]][
'result'] == 'value', 'wrong value of analysis result'
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self._config.set('data_storage', 'report_threshold', '32')
self._config.set('data_storage', 'sanitize_database', 'tmp_sanitize')
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def test_get_results_from_parent_fos(self):
fw = create_test_firmware()
fo = create_test_file_object()
file_name = 'folder/file'
encoded_name = b64_encode(file_name)
fw.processed_analysis[AnalysisPlugin.NAME] = {
'files': {
encoded_name: {
'result': 'value'
}
}
}
fo.virtual_file_path['some_uid'] = [
'some_uid|{}|/{}'.format(fw.uid, file_name)
]
results = {}
routes.FsMetadataRoutesDbInterface.get_results_from_parent_fos(
fw, fo, results)
assert results != {}, 'result should not be empty'
assert file_name in results, 'files missing from result'
assert 'parent_uid' in results[
file_name], 'parent uid missing in result'
assert 'result' in results[file_name], 'analysis result is missing'
assert results[file_name][
'result'] == 'value', 'wrong value of analysis result'
def test_get_file_object_number(self):
result = self.db_interface.get_file_object_number()
self.assertEqual(result, 0)
self.db_interface_backend.add_file_object(self.test_fo)
result = self.db_interface.get_file_object_number(
query={}, zero_on_empty_query=False)
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(
query={'_id': self.test_fo.uid})
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(
query=json.dumps({'_id': self.test_fo.uid}))
self.assertEqual(result, 1)
result = self.db_interface.get_file_object_number(
query={}, zero_on_empty_query=True)
self.assertEqual(result, 0)
result = self.db_interface.get_file_object_number(
query='{}', zero_on_empty_query=True)
self.assertEqual(result, 0)
test_fo_2 = create_test_file_object(
bin_path='get_files_test/testfile2')
self.db_interface_backend.add_file_object(test_fo_2)
result = self.db_interface.get_file_object_number(
query={}, zero_on_empty_query=False)
self.assertEqual(result, 2)
result = self.db_interface.get_file_object_number(
query={'_id': self.test_fo.uid})
self.assertEqual(result, 1)
def test_get_meta_list_of_fo(self):
test_fo = create_test_file_object()
self.db_backend_interface.add_file_object(test_fo)
files = self.db_frontend_interface.file_objects.find()
meta_list = self.db_frontend_interface.get_meta_list(files)
self.assertEqual(meta_list[0][0], test_fo.uid, 'uid of object not correct')
self.assertEqual(meta_list[0][3], 0, 'non existing submission date should lead to 0')
def test_imphash():
fo = create_test_file_object(
bin_path=str(Path(get_test_data_dir(), 'test_executable')))
fo.processed_analysis = {'file_type': {'mime': 'application/x-executable'}}
imphash = get_imphash(fo)
assert isinstance(imphash, str), 'imphash should be a string'
assert len(imphash) == 32, 'imphash does not seem to be an md5'
def test_process_object_this_file(stub_plugin):
test_file = create_test_file_object(bin_path=str(PYLINT_TEST_FILE))
stub_plugin.process_object(test_file)
result = test_file.processed_analysis[stub_plugin.NAME]
assert result['full']
assert result['full'][0]['type'] == 'warning'
assert result['full'][0]['symbol'] == 'unused-import'
def test_generate_file_tree_node(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
child_fo.processed_analysis['file_type'] = {'mime': 'sometype'}
uid = parent_fw.get_uid()
child_fo.virtual_file_path = {
uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]
}
parent_fw.files_included = {child_fo.get_uid()}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
for node in self.db_frontend_interface.generate_file_tree_node(
uid, uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, parent_fw.file_name)
self.assertTrue(node.has_children)
for node in self.db_frontend_interface.generate_file_tree_node(
child_fo.get_uid(), uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, 'folder')
self.assertTrue(node.has_children)
virtual_grand_child = node.get_list_of_child_nodes()[0]
self.assertEqual(virtual_grand_child.type, 'sometype')
self.assertFalse(virtual_grand_child.has_children)
self.assertEqual(virtual_grand_child.name, child_fo.file_name)
def test_get_executable_stats(self):
for i, file_str in enumerate([
'ELF 64-bit LSB executable, x86-64, dynamically linked, for GNU/Linux 2.6.32, not stripped',
'ELF 32-bit MSB executable, MIPS, MIPS32 rel2 version 1 (SYSV), statically linked, not stripped',
'ELF 64-bit LSB executable, x86-64, (SYSV), corrupted section header size',
'ELF 64-bit LSB executable, aarch64, dynamically linked, stripped',
'ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, stripped'
]):
fo = create_test_file_object()
fo.processed_analysis['file_type'] = {'full': file_str}
fo.uid = str(i)
self.db_backend_interface.add_file_object(fo)
stats = self.updater.get_executable_stats().get('executable_stats')
expected = [('big endian', 1, 0.25), ('little endian', 3, 0.75),
('stripped', 1, 0.25), ('not stripped', 2, 0.5),
('32-bit', 1, 0.25), ('64-bit', 3, 0.75),
('dynamically linked', 2, 0.5),
('statically linked', 1, 0.25),
('section info missing', 1, 0.25)]
for (expected_label, expected_count,
expected_percentage), (label, count, percentage,
_) in zip(expected, stats):
assert label == expected_label
assert count == expected_count
assert percentage == expected_percentage
def test_get_specific_fields_for_multiple_entries(self):
test_fw_1 = create_test_firmware(device_name='fw_one', vendor='test_vendor_one')
self.db_backend_interface.add_firmware(test_fw_1)
test_fw_2 = create_test_firmware(device_name='fw_two', vendor='test_vendor_two', bin_path='container/test.7z')
self.db_backend_interface.add_firmware(test_fw_2)
test_fo = create_test_file_object()
self.db_backend_interface.add_file_object(test_fo)
test_uid_list = [test_fw_1.uid, test_fw_2.uid]
result = list(self.db_frontend_interface.get_specific_fields_for_multiple_entries(
uid_list=test_uid_list,
field_dict={'vendor': 1, 'device_name': 1}
))
assert len(result) == 2
assert all(set(entry.keys()) == {'_id', 'vendor', 'device_name'} for entry in result)
result_uids = [entry['_id'] for entry in result]
assert all(uid in result_uids for uid in test_uid_list)
test_uid_list = [test_fw_1.uid, test_fo.uid]
result = list(self.db_frontend_interface.get_specific_fields_for_multiple_entries(
uid_list=test_uid_list,
field_dict={'virtual_file_path': 1}
))
assert len(result) == 2
assert all(set(entry.keys()) == {'_id', 'virtual_file_path'} for entry in result)
result_uids = [entry['_id'] for entry in result]
assert all(uid in result_uids for uid in test_uid_list)
def test_add_file_object_to_firmware():
root = create_test_firmware()
child_fo = create_test_file_object()
root.add_included_file(child_fo)
assert root.uid in child_fo.virtual_file_path.keys(
), 'no virtual file path for root available'
assert child_fo.virtual_file_path[root.uid][0] == join_virtual_path(
root.uid, child_fo.file_path), 'virtual file path not correct'
def test_rest_get_failed_analyses(self):
test_fo = create_test_file_object()
test_fo.processed_analysis['some_analysis'] = {'failed': 'oops'}
self.db_backend.add_file_object(test_fo)
response = json.loads(self.test_client.get('/rest/missing', follow_redirects=True).data.decode())
assert 'failed_analyses' in response
assert 'some_analysis' in response['failed_analyses']
assert test_fo.uid in response['failed_analyses']['some_analysis']
def get_object(self, uid):
if uid == self.fw.uid:
return self.fw
if uid == 'foo':
return self.fo
if uid == 'bar':
fo = create_test_file_object()
fo.virtual_file_path = {'some_uid': ['a|b|c']}
return fo
def test_find_strings(self):
fo = create_test_file_object()
fo.processed_analysis['printable_strings'] = dict(strings=['reasonable', 'still_reasonable', 'n123ot\'(§rea\'§&son##+able'])
fo = self.analysis_plugin.process_object(fo)
results = fo.processed_analysis[self.PLUGIN_NAME]
self.assertTrue(isinstance(results, dict), 'Result of wrong type')
self.assertTrue(results['string_eval'] == ['still_reasonable', 'reasonable', 'n123ot\'(§rea\'§&son##+able'])
def __init__(self):
self.fw = create_test_firmware()
self.fo = create_test_file_object()
self.fo.processed_analysis['file_hashes'] = {
'ssdeep': get_ssdeep(self.fo.binary)
}
self.fw.add_included_file(self.fo)
self.fw.processed_analysis['file_hashes'] = {
'ssdeep': get_ssdeep(self.fw.binary)
}
def test_unpack_status_packed_file(self):
test_fo_packed = create_test_file_object(bin_path='container/test.7z')
test_fo_packed.processed_analysis['unpacker'] = {}
self.unpacker.get_unpack_status(test_fo_packed, [])
result = test_fo_packed.processed_analysis['unpacker']
self.assertGreater(result['entropy'], 0.7, 'entropy not valid')
self.assertEqual(result['summary'], ['packed'], '7z file should be packed')
self.unpacker.VALID_COMPRESSED_FILE_TYPES = ['application/x-7z-compressed']
self.unpacker.get_unpack_status(test_fo_packed, [])
self.assertEqual(test_fo_packed.processed_analysis['unpacker']['summary'], ['unpacked'], 'Unpacking Whitelist does not work')
def test_rest_get_orphaned_objects(self):
test_fo = create_test_file_object()
test_fo.parent_firmware_uids = ['missing_uid']
self.db_backend.add_file_object(test_fo)
response = json.loads(self.test_client.get('/rest/missing', follow_redirects=True).data.decode())
assert 'orphaned_objects' in response
assert response['orphaned_objects'] == {
'missing_uid': ['d558c9339cb967341d701e3184f863d3928973fccdc1d96042583730b5c7b76a_62']
}
def test_get_number_of_total_matches(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
uid = parent_fw.uid
child_fo.virtual_file_path = {uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(uid, child_fo.uid)
self.assertEqual(self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=False), 2)
self.assertEqual(self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True), 1)
def test_find_failed_analyses_with_multiple_analyses(self):
test_fo_1 = create_test_file_object()
test_fo_1.processed_analysis.update({'foo': {'failed': 'some reason'}, 'bar': {'failed': 'another reason'}})
self.db_backend_interface.add_file_object(test_fo_1)
failed_analyses = self.db_frontend_interface.find_failed_analyses()
assert failed_analyses, 'should not be empty'
assert sorted(failed_analyses) == ['bar', 'foo']
assert len(failed_analyses['foo']) == 1 and len(failed_analyses['bar']) == 1
assert test_fo_1.uid in failed_analyses['foo']
def test_rest_download_valid(self):
test_file_object = create_test_file_object()
self.db_backend.add_file_object(test_file_object)
rv = self.test_client.get('/rest/file_object/{}'.format(
test_file_object.uid),
follow_redirects=True)
assert b'hid' in rv.data
assert b'size' in rv.data
def test_process_object_this_file(stub_plugin, monkeypatch):
test_file = create_test_file_object(bin_path=str(PYLINT_TEST_FILE))
monkeypatch.setattr(
'storage.fsorganizer.FSOrganizer.generate_path_from_uid',
lambda _self, _: test_file.file_path)
stub_plugin.process_object(test_file)
result = test_file.processed_analysis[stub_plugin.NAME]
assert result['full']
assert result['full'][0]['type'] == 'warning'
assert result['full'][0]['symbol'] == 'unused-import'
def test_get_parent_uids_from_virtual_path(self):
fo = create_test_file_object()
fo.virtual_file_path = {'fw_uid': ['fw_uid']}
assert len(
plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)) == 0
fo.virtual_file_path = {'some_UID': ['|uid1|uid2|/folder_1/some_file']}
assert 'uid2' in plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)
fo.virtual_file_path = {
'some_UID': [
'|uid1|uid2|/folder_1/some_file',
'|uid1|uid2|/folder_2/some_file'
]
}
result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)
assert 'uid2' in result
assert len(result) == 1
fo.virtual_file_path = {
'uid1':
['|uid1|uid2|/folder_1/some_file', '|uid1|uid3|/some_file']
}
result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)
assert 'uid2' in result
assert 'uid3' in result
assert len(result) == 2
fo.virtual_file_path = {
'uid1': ['|uid1|uid2|/folder_1/some_file'],
'other_UID': ['|other_UID|uid2|/folder_2/some_file']
}
result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)
assert 'uid2' in result
assert len(result) == 1
fo.virtual_file_path = {
'uid1': ['|uid1|uid2|/folder_1/some_file'],
'other_UID': ['|other_UID|uid3|/folder_2/some_file']
}
result = plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)
assert 'uid2' in result
assert 'uid3' in result
assert len(result) == 2
fo.virtual_file_path = {}
assert len(
plugin.FsMetadataDbInterface.get_parent_uids_from_virtual_path(
fo)) == 0
def test_process_object(self):
fo = create_test_file_object()
fo.processed_analysis['ip_and_uri_finder'] = {
'summary':
['1.2.3.4', 'www.example.com', 'www.interesting.receive.org']
}
self.analysis_plugin.process_object(fo)
assert self.PLUGIN_NAME in fo.processed_analysis
assert fo.processed_analysis[self.PLUGIN_NAME]['summary'] == [
'www.interesting.receive.org'
]
def test_add_file_object_path_already_present():
root = create_test_firmware()
child = create_test_file_object()
child.virtual_file_path = {
root.uid: ['{}|some/known/path'.format(root.uid)]
}
root.add_included_file(child)
assert len(child.virtual_file_path.keys()
) == 1, 'there should be just one root object'
assert len(child.virtual_file_path[
root.uid]) == 1, 'number of paths should be one'
def test_get_hid_fo(self):
test_fo = create_test_file_object(bin_path='get_files_test/testfile2')
test_fo.virtual_file_path = {'a': ['|a|/test_file'], 'b': ['|b|/get_files_test/testfile2']}
self.db_backend_interface.add_file_object(test_fo)
result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='b')
self.assertEqual(result, '/get_files_test/testfile2', 'fo hid not correct')
result = self.db_frontend_interface.get_hid(test_fo.uid)
self.assertIsInstance(result, str, 'result is not a string')
self.assertEqual(result[0], '/', 'first character not correct if no root_uid set')
result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='c')
self.assertEqual(result[0], '/', 'first character not correct if invalid root_uid set')
def test_get_number_of_total_matches(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
uid = parent_fw.uid
child_fo.parent_firmware_uids = [uid]
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(uid, child_fo.uid)
assert self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=False, inverted=False) == 2
assert self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True, inverted=False) == 1
assert self.db_frontend_interface.get_number_of_total_matches(query, only_parent_firmwares=True, inverted=True) == 0
def test_get_complete_object_including_all_summaries(self):
self.db_interface_backend.report_threshold = 1024
test_file = create_test_file_object()
self.test_firmware.add_included_file(test_file)
self.db_interface_backend.add_firmware(self.test_firmware)
self.db_interface_backend.add_file_object(test_file)
tmp = self.db_interface.get_complete_object_including_all_summaries(self.test_firmware.uid)
self.assertIsInstance(tmp, Firmware, 'wrong type')
self.assertIn('summary', tmp.processed_analysis['dummy'].keys(), 'summary not found in processed analysis')
self.assertIn('sum a', tmp.processed_analysis['dummy']['summary'], 'summary of original file not included')
self.assertIn('file exclusive sum b', tmp.processed_analysis['dummy']['summary'], 'summary of included file not found')
