class TestTagPropagation(unittest.TestCase):
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.analysis_finished_event = Event()
self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048'
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(config=self._config, pre_analysis=self.backend_interface.add_object, post_analysis=self.count_analysis_finished_event)
self._tagging_scheduler = TaggingDaemon(analysis_scheduler=self._analysis_scheduler)
self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.start_analysis_of_object)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
if fw_object.uid == self.uid_of_key_file and 'crypto_material' in fw_object.processed_analysis:
sleep(1)
self.analysis_finished_event.set()
def _wait_for_empty_tag_queue(self):
while not self._analysis_scheduler.tag_queue.empty():
sleep(0.1)
def tearDown(self):
self._unpack_scheduler.shutdown()
self._tagging_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_run_analysis_with_tag(self):
test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir()))
test_fw.release_date = '2017-01-01'
test_fw.scheduled_analysis = ['crypto_material']
self._unpack_scheduler.add_task(test_fw)
assert self.analysis_finished_event.wait(timeout=20)
processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material'])
assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis'
self._wait_for_empty_tag_queue()
processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material'])
assert processed_fw.analysis_tags, 'tags not propagated properly'
assert processed_fw.analysis_tags['crypto_material']['private_key_inside']
class TestFileAddition(unittest.TestCase):
@patch('unpacker.unpack.FS_Organizer', MockFSOrganizer)
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(
config=self._config, callback=self.trigger_compare_finished_event)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_analysis(fw_object)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 2 * 2: # 2 container with 3 files each and 2 plugins
self.analysis_finished_event.set()
def trigger_compare_finished_event(self):
self.compare_finished_event.set()
def tearDown(self):
self._compare_scheduler.shutdown()
self._unpack_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
@staticmethod
def _expected_result():
return {
'File_Coverage': {
'files_in_common': {
'all': [],
'collapse': False
}
},
'Software': {
'Compare Skipped': {
'all':
'Required analysis not present: [\'software_components\', \'software_components\']'
}
}
}
class TestRestFirmware(TestAcceptanceBase):
def setUp(self):
super().setUp()
self.analysis_finished_event = Event()
self.elements_finished_analyzing = Value('i', 0)
self.db_backend_service = BackEndDbInterface(config=self.config)
self._start_backend(post_analysis=self._analysis_callback)
self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787'
time.sleep(2) # wait for systems to start
def tearDown(self):
self._stop_backend()
self.db_backend_service.shutdown()
super().tearDown()
def _analysis_callback(self, fo):
self.db_backend_service.add_analysis(fo)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value > 5:
self.analysis_finished_event.set()
def _rest_upload_firmware(self):
testfile_path = os.path.join(get_test_data_dir(), 'container/test.zip')
with open(testfile_path, 'rb') as fp:
file_content = fp.read()
data = {
'binary': standard_b64encode(file_content).decode(),
'file_name': 'test.zip',
'device_name': 'test_device',
'device_class': 'test_class',
'firmware_version': '1.0',
'vendor': 'test_vendor',
'release_date': '01.01.1970',
'tags': '',
'requested_analysis_systems': ['software_components']
}
rv = self.test_client.put('/rest/firmware', data=json.dumps(data), follow_redirects=True)
self.assertIn(b'"status": 0', rv.data, 'rest upload not successful')
self.assertIn(self.test_container_uid.encode(), rv.data, 'uid not found in rest upload reply')
def _rest_get_analysis_result(self):
rv = self.test_client.get('/rest/firmware/{}'.format(self.test_container_uid), follow_redirects=True)
self.assertIn(b'analysis_date', rv.data, 'rest analysis download not successful')
self.assertIn(b'software_components', rv.data, 'rest analysis not successful')
def _rest_search(self):
rv = self.test_client.get('/rest/firmware?query={}'.format(urllib.parse.quote('{"device_class": "test_class"}')), follow_redirects=True)
self.assertIn(self.test_container_uid.encode(), rv.data, 'test firmware not found in rest search')
def _rest_search_fw_only(self):
query = json.dumps({'sha256': self.test_container_uid.split('_')[0]})
rv = self.test_client.get('/rest/firmware?query={}'.format(urllib.parse.quote(query)), follow_redirects=True)
self.assertIn(self.test_container_uid.encode(), rv.data, 'test firmware not found in rest search')
def _rest_update_analysis_bad_analysis(self):
rv = self.test_client.put('/rest/firmware/{}?update={}'.format(self.test_container_uid, urllib.parse.quote('["unknown_system"]')), follow_redirects=True)
self.assertIn('Unknown analysis system'.encode(), rv.data, "rest analysis update should break on request of non existing system")
def _rest_update_analysis_success(self):
rv = self.test_client.put('/rest/firmware/{}?update={}'.format(self.test_container_uid, urllib.parse.quote(json.dumps(['crypto_material']))), follow_redirects=True)
self.assertNotIn(b'error_message', rv.data, 'Error on update request')
def _rest_check_new_analysis_exists(self):
rv = self.test_client.get('/rest/firmware/{}'.format(self.test_container_uid), follow_redirects=True)
response_data = json.loads(rv.data.decode())
assert response_data['firmware']['analysis']['crypto_material']
assert response_data['firmware']['analysis']['crypto_material']['analysis_date'] > response_data['firmware']['analysis']['software_components']['analysis_date']
def test_run_from_upload_to_show_analysis_and_search(self):
self._rest_upload_firmware()
self.analysis_finished_event.wait(timeout=15)
self.elements_finished_analyzing.value = 0
self.analysis_finished_event.clear()
self._rest_get_analysis_result()
self._rest_search()
self._rest_search_fw_only()
self._rest_update_analysis_bad_analysis()
self._rest_update_analysis_success()
self.analysis_finished_event.wait(timeout=10)
self._rest_check_new_analysis_exists()
class TestStorageDbInterfaceBackend(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._config = get_config_for_testing(TMP_DIR)
cls.mongo_server = MongoMgr(config=cls._config)
def setUp(self):
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def tearDown(self):
self.db_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_interface_backend.shutdown()
self.db_interface.shutdown()
gc.collect()
@classmethod
def tearDownClass(cls):
cls.mongo_server.shutdown()
TMP_DIR.cleanup()
def _get_all_firmware_uids(self):
uid_list = []
tmp = self.db_interface.firmwares.find()
for item in tmp:
uid_list.append(item['_id'])
return uid_list
def test_add_firmware(self):
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertGreater(len(self._get_all_firmware_uids()), 0,
'No entry added to DB')
recoverd_firmware_entry = self.db_interface_backend.firmwares.find_one(
)
self.assertAlmostEqual(recoverd_firmware_entry['submission_date'],
time(),
msg='submission time not set correctly',
delta=5.0)
def test_add_and_get_firmware(self):
self.db_interface_backend.add_firmware(self.test_firmware)
result_backend = self.db_interface_backend.get_firmware(
self.test_firmware.uid)
self.assertIsNotNone(result_backend.binary,
'binary not set in backend result')
result_common = self.db_interface.get_firmware(self.test_firmware.uid)
self.assertIsNone(result_common.binary, 'binary set in common result')
self.assertEqual(result_common.size, 787,
'file size not correct in common')
self.assertIsInstance(result_common.tags, dict,
'tag field type not correct')
def test_add_and_get_file_object(self):
self.db_interface_backend.add_file_object(self.test_fo)
result_backend = self.db_interface_backend.get_file_object(
self.test_fo.uid)
self.assertIsNotNone(result_backend.binary,
'binary not set in backend result')
result_common = self.db_interface.get_file_object(self.test_fo.uid)
self.assertIsNone(result_common.binary, 'binary set in common result')
self.assertEqual(result_common.size, 62,
'file size not correct in common')
def test_update_firmware(self):
first_dict = {
'stub_plugin': {
'result': 0
},
'other_plugin': {
'field': 'day'
}
}
second_dict = {'stub_plugin': {'result': 1}}
self.test_firmware.processed_analysis = first_dict
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertEqual(
0,
self.db_interface.get_object(
self.test_firmware.uid).processed_analysis['stub_plugin']
['result'])
self.test_firmware.processed_analysis = second_dict
self.db_interface_backend.add_firmware(self.test_firmware)
self.assertEqual(
1,
self.db_interface.get_object(
self.test_firmware.uid).processed_analysis['stub_plugin']
['result'])
self.assertIn(
'other_plugin',
self.db_interface.get_object(
self.test_firmware.uid).processed_analysis.keys())
def test_update_file_object(self):
first_dict = {'other_plugin': {'result': 0}}
second_dict = {'stub_plugin': {'result': 1}}
self.test_fo.processed_analysis = first_dict
self.test_fo.files_included = {'file a', 'file b'}
self.db_interface_backend.add_file_object(self.test_fo)
self.test_fo.processed_analysis = second_dict
self.test_fo.files_included = {'file b', 'file c'}
self.db_interface_backend.add_file_object(self.test_fo)
received_object = self.db_interface.get_object(self.test_fo.uid)
self.assertEqual(
0, received_object.processed_analysis['other_plugin']['result'])
self.assertEqual(
1, received_object.processed_analysis['stub_plugin']['result'])
self.assertEqual(3, len(received_object.files_included))
def test_add_and_get_object_including_comment(self):
comment, author, date, uid = 'this is a test comment!', 'author', '1473431685', self.test_fo.uid
self.test_fo.comments.append({
'time': str(date),
'author': author,
'comment': comment
})
self.db_interface_backend.add_file_object(self.test_fo)
retrieved_comment = self.db_interface.get_object(uid).comments[0]
self.assertEqual(author, retrieved_comment['author'])
self.assertEqual(comment, retrieved_comment['comment'])
self.assertEqual(date, retrieved_comment['time'])
def test_update_analysis_tag_no_firmware(self):
self.db_interface_backend.add_file_object(self.test_fo)
tag = {'value': 'yay', 'color': 'default', 'propagate': True}
self.db_interface_backend.update_analysis_tags(self.test_fo.uid,
plugin_name='dummy',
tag_name='some_tag',
tag=tag)
processed_fo = self.db_interface_backend.get_object(self.test_fo.uid)
assert not processed_fo.analysis_tags
def test_update_analysis_tag_uid_not_found(self):
self.db_interface_backend.update_analysis_tags(self.test_fo.uid,
plugin_name='dummy',
tag_name='some_tag',
tag='should not matter')
assert not self.db_interface_backend.get_object(self.test_fo.uid)
def test_update_analysis_tag_bad_tag(self):
self.db_interface_backend.add_firmware(self.test_firmware)
self.db_interface_backend.update_analysis_tags(self.test_firmware.uid,
plugin_name='dummy',
tag_name='some_tag',
tag='bad_tag')
processed_firmware = self.db_interface_backend.get_object(
self.test_firmware.uid)
assert not processed_firmware.analysis_tags
def test_update_analysis_tag_success(self):
self.db_interface_backend.add_firmware(self.test_firmware)
tag = {'value': 'yay', 'color': 'default', 'propagate': True}
self.db_interface_backend.update_analysis_tags(self.test_firmware.uid,
plugin_name='dummy',
tag_name='some_tag',
tag=tag)
processed_firmware = self.db_interface_backend.get_object(
self.test_firmware.uid)
assert processed_firmware.analysis_tags
assert processed_firmware.analysis_tags['dummy']['some_tag'] == tag
def test_add_analysis_firmware(self):
self.db_interface_backend.add_object(self.test_firmware)
before = self.db_interface_backend.get_object(
self.test_firmware.uid).processed_analysis
self.test_firmware.processed_analysis['foo'] = {'bar': 5}
self.db_interface_backend.add_analysis(self.test_firmware)
after = self.db_interface_backend.get_object(
self.test_firmware.uid).processed_analysis
assert before != after
assert 'foo' not in before
assert 'foo' in after
assert after['foo'] == {'bar': 5}
def test_add_analysis_file_object(self):
self.db_interface_backend.add_object(self.test_fo)
self.test_fo.processed_analysis['foo'] = {'bar': 5}
self.db_interface_backend.add_analysis(self.test_fo)
analysis = self.db_interface_backend.get_object(
self.test_fo.uid).processed_analysis
assert 'foo' in analysis
assert analysis['foo'] == {'bar': 5}
def test_crash_add_analysis(self):
with self.assertRaises(RuntimeError):
self.db_interface_backend.add_analysis(dict())
with self.assertRaises(AttributeError):
self.db_interface_backend._update_analysis(dict(), 'dummy', dict())
class TestStorageDbInterfaceFrontend(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._config = get_config_for_testing(TMP_DIR)
cls.mongo_server = MongoMgr(config=cls._config)
def setUp(self):
self.db_frontend_interface = FrontEndDbInterface(config=self._config)
self.db_backend_interface = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
def tearDown(self):
self.db_frontend_interface.shutdown()
self.db_backend_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
self.db_backend_interface.shutdown()
gc.collect()
@classmethod
def tearDownClass(cls):
cls.mongo_server.shutdown()
TMP_DIR.cleanup()
def test_regression_meta_list(self):
assert self.test_firmware.processed_analysis.pop('unpacker')
self.db_backend_interface.add_firmware(self.test_firmware)
list_of_firmwares = self.db_frontend_interface.get_meta_list()
assert 'NOP' in list_of_firmwares.pop()[2]
def test_get_meta_list(self):
self.db_backend_interface.add_firmware(self.test_firmware)
list_of_firmwares = self.db_frontend_interface.get_meta_list()
test_output = list_of_firmwares.pop()
self.assertEqual(test_output[1],
'test_vendor test_router - 0.1 (Router)',
'Firmware not successfully received')
self.assertIsInstance(test_output[2], dict, 'tag field is not a dict')
def test_get_meta_list_of_fo(self):
test_fo = create_test_file_object()
self.db_backend_interface.add_file_object(test_fo)
files = self.db_frontend_interface.file_objects.find()
meta_list = self.db_frontend_interface.get_meta_list(files)
self.assertEqual(meta_list[0][0], test_fo.uid,
'uid of object not correct')
self.assertEqual(meta_list[0][3], 0,
'non existing submission date should lead to 0')
def test_get_hid_firmware(self):
self.db_backend_interface.add_firmware(self.test_firmware)
result = self.db_frontend_interface.get_hid(self.test_firmware.uid)
self.assertEqual(result, 'test_vendor test_router - 0.1 (Router)',
'fw hid not correct')
def test_get_hid_fo(self):
test_fo = create_test_file_object(bin_path='get_files_test/testfile2')
test_fo.virtual_file_path = {
'a': ['|a|/test_file'],
'b': ['|b|/get_files_test/testfile2']
}
self.db_backend_interface.add_file_object(test_fo)
result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='b')
self.assertEqual(result, '/get_files_test/testfile2',
'fo hid not correct')
result = self.db_frontend_interface.get_hid(test_fo.uid)
self.assertIsInstance(result, str, 'result is not a string')
self.assertEqual(result[0], '/',
'first character not correct if no root_uid set')
result = self.db_frontend_interface.get_hid(test_fo.uid, root_uid='c')
self.assertEqual(
result[0], '/',
'first character not correct if invalid root_uid set')
def test_get_file_name(self):
self.db_backend_interface.add_firmware(self.test_firmware)
result = self.db_frontend_interface.get_file_name(
self.test_firmware.uid)
self.assertEqual(result, 'test.zip', 'name not correct')
def test_get_hid_invalid_uid(self):
result = self.db_frontend_interface.get_hid('foo')
self.assertEqual(result, '',
'invalid uid should result in empty string')
def test_get_firmware_attribute_list(self):
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertEqual(self.db_frontend_interface.get_device_class_list(),
['Router'])
self.assertEqual(self.db_frontend_interface.get_vendor_list(),
['test_vendor'])
self.assertEqual(
self.db_frontend_interface.get_firmware_attribute_list(
'device_name', {
'vendor': 'test_vendor',
'device_class': 'Router'
}), ['test_router'])
self.assertEqual(
self.db_frontend_interface.get_firmware_attribute_list('version'),
['0.1'])
self.assertEqual(self.db_frontend_interface.get_device_name_dict(),
{'Router': {
'test_vendor': ['test_router']
}})
def test_get_data_for_nice_list(self):
uid_list = [self.test_firmware.uid]
self.db_backend_interface.add_firmware(self.test_firmware)
nice_list_data = self.db_frontend_interface.get_data_for_nice_list(
uid_list, uid_list[0])
self.assertEqual(
sorted([
'size', 'virtual_file_paths', 'uid', 'mime-type',
'files_included'
]), sorted(nice_list_data[0].keys()))
self.assertEqual(nice_list_data[0]['uid'], self.test_firmware.uid)
def test_generic_search(self):
self.db_backend_interface.add_firmware(self.test_firmware)
# str input
result = self.db_frontend_interface.generic_search(
'{"file_name": "test.zip"}')
self.assertEqual(result, [self.test_firmware.uid],
'Firmware not successfully received')
# dict input
result = self.db_frontend_interface.generic_search(
{'file_name': 'test.zip'})
self.assertEqual(result, [self.test_firmware.uid],
'Firmware not successfully received')
def test_all_uids_found_in_database(self):
self.db_backend_interface.client.drop_database(
self._config.get('data_storage', 'main_database'))
uid_list = [self.test_firmware.uid]
self.assertFalse(
self.db_frontend_interface.all_uids_found_in_database(uid_list))
self.db_backend_interface.add_firmware(self.test_firmware)
self.assertTrue(
self.db_frontend_interface.all_uids_found_in_database(
[self.test_firmware.uid]))
def test_get_x_last_added_firmwares(self):
self.assertEqual(self.db_frontend_interface.get_last_added_firmwares(),
[], 'empty db should result in empty list')
test_fw_one = create_test_firmware(device_name='fw_one')
self.db_backend_interface.add_firmware(test_fw_one)
test_fw_two = create_test_firmware(device_name='fw_two',
bin_path='container/test.7z')
self.db_backend_interface.add_firmware(test_fw_two)
test_fw_three = create_test_firmware(device_name='fw_three',
bin_path='container/test.cab')
self.db_backend_interface.add_firmware(test_fw_three)
result = self.db_frontend_interface.get_last_added_firmwares(limit_x=2)
self.assertEqual(len(result), 2, 'Number of results should be 2')
self.assertEqual(result[0][0], test_fw_three.uid,
'last firmware is not first entry')
self.assertEqual(result[1][0], test_fw_two.uid,
'second last firmware is not the second entry')
def test_generate_file_tree_level(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
child_fo.processed_analysis['file_type'] = {'mime': 'sometype'}
uid = parent_fw.uid
child_fo.virtual_file_path = {
uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]
}
parent_fw.files_included = {child_fo.uid}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
for node in self.db_frontend_interface.generate_file_tree_level(
uid, uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, parent_fw.file_name)
self.assertTrue(node.has_children)
for node in self.db_frontend_interface.generate_file_tree_level(
child_fo.uid, uid):
self.assertIsInstance(node, FileTreeNode)
self.assertEqual(node.name, 'folder')
self.assertTrue(node.has_children)
virtual_grand_child = node.get_list_of_child_nodes()[0]
self.assertEqual(virtual_grand_child.type, 'sometype')
self.assertFalse(virtual_grand_child.has_children)
self.assertEqual(virtual_grand_child.name, child_fo.file_name)
def test_get_number_of_total_matches(self):
parent_fw = create_test_firmware()
child_fo = create_test_file_object()
uid = parent_fw.uid
child_fo.virtual_file_path = {
uid: ['|{}|/folder/{}'.format(uid, child_fo.file_name)]
}
self.db_backend_interface.add_object(parent_fw)
self.db_backend_interface.add_object(child_fo)
query = '{{"$or": [{{"_id": "{}"}}, {{"_id": "{}"}}]}}'.format(
uid, child_fo.uid)
self.assertEqual(
self.db_frontend_interface.get_number_of_total_matches(
query, only_parent_firmwares=False), 2)
self.assertEqual(
self.db_frontend_interface.get_number_of_total_matches(
query, only_parent_firmwares=True), 1)
def test_get_other_versions_of_firmware(self):
parent_fw1 = create_test_firmware(version='1')
self.db_backend_interface.add_object(parent_fw1)
parent_fw2 = create_test_firmware(version='2',
bin_path='container/test.7z')
self.db_backend_interface.add_object(parent_fw2)
parent_fw3 = create_test_firmware(version='3',
bin_path='container/test.cab')
self.db_backend_interface.add_object(parent_fw3)
other_versions = self.db_frontend_interface.get_other_versions_of_firmware(
parent_fw1)
self.assertEqual(len(other_versions), 2,
'wrong number of other versions')
self.assertIn({'_id': parent_fw2.uid, 'version': '2'}, other_versions)
self.assertIn({'_id': parent_fw3.uid, 'version': '3'}, other_versions)
other_versions = self.db_frontend_interface.get_other_versions_of_firmware(
parent_fw2)
self.assertIn({'_id': parent_fw3.uid, 'version': '3'}, other_versions)
def test_get_specific_fields_for_multiple_entries(self):
test_fw_1 = create_test_firmware(device_name='fw_one',
vendor='test_vendor_one')
self.db_backend_interface.add_firmware(test_fw_1)
test_fw_2 = create_test_firmware(device_name='fw_two',
vendor='test_vendor_two',
bin_path='container/test.7z')
self.db_backend_interface.add_firmware(test_fw_2)
test_fo = create_test_file_object()
self.db_backend_interface.add_file_object(test_fo)
test_uid_list = [test_fw_1.uid, test_fw_2.uid]
result = list(
self.db_frontend_interface.
get_specific_fields_for_multiple_entries(uid_list=test_uid_list,
field_dict={
'vendor': 1,
'device_name': 1
}))
assert len(result) == 2
assert all(
set(entry.keys()) == {'_id', 'vendor', 'device_name'}
for entry in result)
result_uids = [entry['_id'] for entry in result]
assert all(uid in result_uids for uid in test_uid_list)
test_uid_list = [test_fw_1.uid, test_fo.uid]
result = list(
self.db_frontend_interface.
get_specific_fields_for_multiple_entries(
uid_list=test_uid_list, field_dict={'virtual_file_path': 1}))
assert len(result) == 2
assert all(
set(entry.keys()) == {'_id', 'virtual_file_path'}
for entry in result)
result_uids = [entry['_id'] for entry in result]
assert all(uid in result_uids for uid in test_uid_list)
def test_find_missing_files(self):
test_fw_1 = create_test_firmware()
test_fw_1.files_included.add('uid1234')
self.db_backend_interface.add_firmware(test_fw_1)
missing_files = self.db_frontend_interface.find_missing_files()
assert test_fw_1.uid in missing_files
assert missing_files[test_fw_1.uid] == {'uid1234'}
test_fo = create_test_file_object()
test_fo.uid = 'uid1234'
self.db_backend_interface.add_file_object(test_fo)
missing_files = self.db_frontend_interface.find_missing_files()
assert missing_files == {}
def test_find_missing_analyses(self):
test_fw_1 = create_test_firmware()
test_fo = create_test_file_object()
test_fw_1.files_included.add(test_fo.uid)
test_fo.virtual_file_path = {test_fw_1.uid: ['|foo|bar|']}
self.db_backend_interface.add_firmware(test_fw_1)
self.db_backend_interface.add_file_object(test_fo)
missing_analyses = self.db_frontend_interface.find_missing_analyses()
assert missing_analyses == {}
test_fw_1.processed_analysis['foobar'] = {'foo': 'bar'}
self.db_backend_interface.add_analysis(test_fw_1)
missing_analyses = self.db_frontend_interface.find_missing_analyses()
assert test_fw_1.uid in missing_analyses
assert missing_analyses[test_fw_1.uid] == {test_fo.uid}
class TestRestFirmware(TestAcceptanceBase):
def setUp(self):
super().setUp()
self.analysis_finished_event = Event()
self.elements_finished_analyzing = Value('i', 0)
self.db_backend_service = BackEndDbInterface(config=self.config)
self._start_backend(post_analysis=self._analysis_callback)
self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787'
time.sleep(2) # wait for systems to start
def tearDown(self):
self._stop_backend()
self.db_backend_service.shutdown()
super().tearDown()
def _analysis_callback(self, fo):
self.db_backend_service.add_analysis(fo)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == 4 * 3: # container including 3 files times 3 plugins
self.analysis_finished_event.set()
def _rest_upload_firmware(self):
data = get_firmware_for_rest_upload_test()
rv = self.test_client.put('/rest/firmware',
json=data,
follow_redirects=True)
assert b'"status": 0' in rv.data, 'rest upload not successful'
assert self.test_container_uid.encode(
) in rv.data, 'uid not found in rest upload reply'
def _rest_get_analysis_result(self):
rv = self.test_client.get(f'/rest/firmware/{self.test_container_uid}',
follow_redirects=True)
assert b'analysis_date' in rv.data, 'rest analysis download not successful'
assert b'software_components' in rv.data, 'rest analysis not successful'
assert b'"device_part": "test_part' in rv.data, 'device part not present'
def _rest_search(self):
query = urllib.parse.quote('{"device_class": "test_class"}')
rv = self.test_client.get(f'/rest/firmware?query={query}',
follow_redirects=True)
assert self.test_container_uid.encode(
) in rv.data, 'test firmware not found in rest search'
def _rest_search_fw_only(self):
query = json.dumps({'sha256': self.test_container_uid.split('_')[0]})
rv = self.test_client.get(
f'/rest/firmware?query={urllib.parse.quote(query)}',
follow_redirects=True)
assert self.test_container_uid.encode(
) in rv.data, 'test firmware not found in rest search'
def _rest_update_analysis_bad_analysis(self):
query = urllib.parse.quote('["unknown_system"]')
rv = self.test_client.put(
f'/rest/firmware/{self.test_container_uid}?update={query}',
follow_redirects=True)
assert 'Unknown analysis system'.encode(
) in rv.data, 'rest analysis update should break on request of non existing system'
def _rest_update_analysis_success(self):
update = urllib.parse.quote(json.dumps(['crypto_material']))
rv = self.test_client.put(
f'/rest/firmware/{self.test_container_uid}?update={update}',
follow_redirects=True)
assert b'error_message' not in rv.data, 'Error on update request'
def _rest_check_new_analysis_exists(self):
rv = self.test_client.get(f'/rest/firmware/{self.test_container_uid}',
follow_redirects=True)
response_data = json.loads(rv.data.decode())
assert response_data['firmware']['analysis']['crypto_material']
assert response_data['firmware']['analysis']['crypto_material'][
'analysis_date'] > response_data['firmware']['analysis'][
'software_components']['analysis_date']
def test_run_from_upload_to_show_analysis_and_search(self):
self._rest_upload_firmware()
self.analysis_finished_event.wait(timeout=15)
self.elements_finished_analyzing.value = 4 * 2 # only one plugin to update so we offset with 4 times 2 plugins
self.analysis_finished_event.clear()
self._rest_get_analysis_result()
self._rest_search()
self._rest_search_fw_only()
self._rest_update_analysis_bad_analysis()
self._rest_update_analysis_success()
self.analysis_finished_event.wait(timeout=10)
self._rest_check_new_analysis_exists()