class TestIntegrationRestDownloadFirmware(TestAcceptanceBase):
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend = BackEndDbInterface(config=self.config)
time.sleep(2) # wait for systems to start
def tearDown(self):
self.db_backend.shutdown()
self._stop_backend()
super().tearDown()
def _rest_search(self):
rv = self.test_client.get('/rest/firmware?query={}'.format(urllib.parse.quote('{"device_class": "test class"}')), follow_redirects=True)
self.assertIn(self.test_fw.uid.encode(), rv.data, "test firmware not found in rest search")
def _rest_download(self):
rv = self.test_client.get('/rest/binary/{}'.format(self.test_fw.uid), follow_redirects=True)
self.assertIn(standard_b64encode(self.test_fw.binary), rv.data, "rest download response incorrect")
self.assertIn('"file_name": "{}"'.format(self.test_fw.file_name).encode(), rv.data, "rest download response incorrect")
self.assertIn('"SHA256": "{}"'.format(self.test_fw.sha256).encode(), rv.data, "rest download response incorrect")
def test_run_from_upload_to_show_analysis(self):
self.test_fw = create_test_firmware(device_class="test class", device_name="test device", vendor="test vendor")
self.db_backend.add_firmware(self.test_fw)
self._rest_search()
self._rest_download()
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self._config.set('data_storage', 'report_threshold', '32')
self._config.set('data_storage', 'sanitize_database', 'tmp_sanitize')
self.mongo_server = MongoMgr(config=self._config)
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def setUp(self):
self._config = get_config_for_testing(TMP_DIR)
self.mongo_server = MongoMgr(config=self._config)
self.db_frontend_editing = FrontendEditingDbInterface(
config=self._config)
self.db_frontend_interface = FrontEndDbInterface(config=self._config)
self.db_backend_interface = BackEndDbInterface(config=self._config)
def setUp(self):
super().setUp()
self.analysis_finished_event = Event()
self.elements_finished_analyzing = Value('i', 0)
self.db_backend_service = BackEndDbInterface(config=self.config)
self._start_backend(post_analysis=self._analysis_callback)
time.sleep(2) # wait for systems to start
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend_interface = BackEndDbInterface(self.config)
self.test_fw = create_test_firmware(device_name='test_fw')
self.test_fw.release_date = '2001-02-03'
self.db_backend_interface.add_firmware(self.test_fw)
class TestRestFileObject(RestTestBase):
def setUp(self):
super().setUp()
self.db_backend = BackEndDbInterface(config=self.config)
def tearDown(self):
self.db_backend.shutdown()
super().tearDown()
def test_rest_download_valid(self):
test_file_object = create_test_file_object()
self.db_backend.add_file_object(test_file_object)
rv = self.test_client.get('/rest/file_object/{}'.format(
test_file_object.uid),
follow_redirects=True)
assert b'hid' in rv.data
assert b'size' in rv.data
def test_rest_request_multiple_file_objects(self):
rv = self.test_client.get('/rest/file_object', follow_redirects=True)
assert b'uids' in rv.data
assert b'status:" 1' not in rv.data
def test_rest_download_invalid_uid(self):
rv = self.test_client.get('/rest/file_object/invalid%20uid',
follow_redirects=True)
assert b'No file object with UID invalid uid' in rv.data
def test_rest_download_invalid_data(self):
rv = self.test_client.get('/rest/file_object/', follow_redirects=True)
assert b'404 Not Found' in rv.data
class TestTagPropagation(unittest.TestCase):
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.analysis_finished_event = Event()
self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048'
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(
config=self._config,
post_analysis=self.count_analysis_finished_event)
self._tagging_scheduler = TaggingDaemon(
analysis_scheduler=self._analysis_scheduler)
self._unpack_scheduler = UnpackingScheduler(
config=self._config, post_unpack=self._analysis_scheduler.add_task)
def count_analysis_finished_event(self, fw_object):
self.backend_interface.add_object(fw_object)
if fw_object.uid == self.uid_of_key_file:
self.analysis_finished_event.set()
def _wait_for_empty_tag_queue(self):
while not self._analysis_scheduler.tag_queue.empty():
sleep(0.1)
def tearDown(self):
self._unpack_scheduler.shutdown()
self._tagging_scheduler.shutdown()
self._analysis_scheduler.shutdown()
clean_test_database(self._config, get_database_names(self._config))
self._mongo_server.shutdown()
self._tmp_dir.cleanup()
gc.collect()
def test_run_analysis_with_tag(self):
test_fw = Firmware(
file_path='{}/container/with_key.7z'.format(get_test_data_dir()))
test_fw.release_date = '2017-01-01'
test_fw.scheduled_analysis = ['crypto_material']
self._unpack_scheduler.add_task(test_fw)
assert self.analysis_finished_event.wait(timeout=20)
processed_fo = self.backend_interface.get_object(
self.uid_of_key_file, analysis_filter=['crypto_material'])
assert processed_fo.processed_analysis['crypto_material'][
'tags'], 'no tags set in analysis'
self._wait_for_empty_tag_queue()
processed_fw = self.backend_interface.get_object(
test_fw.uid, analysis_filter=['crypto_material'])
assert processed_fw.analysis_tags, 'tags not propagated properly'
assert processed_fw.analysis_tags['crypto_material'][
'private_key_inside']
def setUp(self):
super().setUp()
self.analysis_finished_event = Event()
self.elements_finished_analyzing = Value('i', 0)
self.db_backend_service = BackEndDbInterface(config=self.config)
self._start_backend(post_analysis=self._analysis_callback)
self.test_container_uid = '418a54d78550e8584291c96e5d6168133621f352bfc1d43cf84e81187fef4962_787'
time.sleep(2) # wait for systems to start
def setUp(self):
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.db_interface_compare = CompareDbInterface(config=self._config)
self.db_interface_admin = AdminDbInterface(config=self._config)
self.fw_one = create_test_firmware()
self.fw_two = create_test_firmware()
self.fw_two.set_binary(b'another firmware')
self.compare_dict = self._create_compare_dict()
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.analysis_finished_event = Event()
self.uid_of_key_file = '530bf2f1203b789bfe054d3118ebd29a04013c587efd22235b3b9677cee21c0e_2048'
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(config=self._config, pre_analysis=self.backend_interface.add_object, post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.start_analysis_of_object)
class TestAcceptanceBaseWithDb(TestAcceptanceBase):
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend = BackEndDbInterface(config=self.config)
time.sleep(2) # wait for systems to start
def tearDown(self):
self.db_backend.shutdown()
self._stop_backend()
super().tearDown()
def setUp(self):
self._tmp_dir = TemporaryDirectory()
self._config = initialize_config(self._tmp_dir)
self.elements_finished_analyzing = Value('i', 0)
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self._mongo_server = MongoMgr(config=self._config, auth=False)
self.backend_interface = BackEndDbInterface(config=self._config)
self._analysis_scheduler = AnalysisScheduler(config=self._config, post_analysis=self.count_analysis_finished_event)
self._unpack_scheduler = UnpackingScheduler(config=self._config, post_unpack=self._analysis_scheduler.add_task)
self._compare_scheduler = CompareScheduler(config=self._config, callback=self.trigger_compare_finished_event)
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend_interface = BackEndDbInterface(self.config)
self.parent_fw = create_test_firmware()
self.child_fo = create_test_file_object()
uid = self.parent_fw.uid
self.child_fo.virtual_file_path = {uid: ['|{}|/folder/{}'.format(uid, self.child_fo.file_name)]}
self.db_backend_interface.add_object(self.parent_fw)
self.child_fo.processed_analysis['unpacker'] = {}
self.child_fo.processed_analysis['unpacker']['plugin_used'] = 'test'
self.child_fo.processed_analysis['file_type']['mime'] = 'some_type'
self.db_backend_interface.add_object(self.child_fo)
class TestRestDownload(RestTestBase):
def setup(self):
super().setup()
self.db_interface = BackEndDbInterface(self.config)
self.test_queue = Queue()
def teardown(self):
self.test_queue.close()
self.db_interface.shutdown()
super().teardown()
def test_rest_download_valid(self):
backend_binding = InterComBackEndBinding(
config=self.config,
analysis_service=test_backend_scheduler.AnalysisServiceMock(),
compare_service=test_backend_scheduler.ServiceMock(
self.test_queue),
unpacking_service=test_backend_scheduler.ServiceMock(
self.test_queue))
test_firmware = create_test_firmware(device_class='test class',
device_name='test device',
vendor='test vendor')
store_binary_on_file_system(self.tmp_dir.name, test_firmware)
self.db_interface.add_firmware(test_firmware)
try:
rv = self.test_client.get('/rest/binary/{}'.format(
test_firmware.uid),
follow_redirects=True)
finally:
backend_binding.shutdown()
assert standard_b64encode(test_firmware.binary) in rv.data
assert '"file_name": "{}"'.format(
test_firmware.file_name).encode() in rv.data
assert '"SHA256": "{}"'.format(
test_firmware.sha256).encode() in rv.data
def test_rest_download_invalid_uid(self):
rv = self.test_client.get('/rest/binary/not%20existing%20uid',
follow_redirects=True)
assert b'No firmware with UID not existing uid found in database' in rv.data
def test_rest_download_invalid_data(self):
rv = self.test_client.get('/rest/binary/', follow_redirects=True)
assert b'404 Not Found' in rv.data
def test_scheduler(test_config, finished_event, intermediate_event):
interface = BackEndDbInterface(config=test_config)
elements_finished = Value('i', 0)
def count_pre_analysis(file_object):
interface.add_object(file_object)
elements_finished.value += 1
if elements_finished.value == 16:
finished_event.set()
elif elements_finished.value == 8:
intermediate_event.set()
analyzer = AnalysisScheduler(test_config,
pre_analysis=count_pre_analysis,
db_interface=interface)
unpacker = UnpackingScheduler(
config=test_config, post_unpack=analyzer.start_analysis_of_object)
intercom = InterComBackEndBinding(config=test_config,
analysis_service=analyzer,
unpacking_service=unpacker,
compare_service=MockScheduler())
yield unpacker
intercom.shutdown()
unpacker.shutdown()
analyzer.shutdown()
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend_interface = BackEndDbInterface(self.config)
self.parent_fw = create_test_firmware()
self.child_fo = create_test_file_object()
uid = self.parent_fw.uid
self.child_fo.parent_firmware_uids = [uid]
self.db_backend_interface.add_object(self.parent_fw)
self.child_fo.processed_analysis['unpacker'] = {}
self.child_fo.processed_analysis['unpacker']['plugin_used'] = 'test'
self.child_fo.processed_analysis['file_type']['mime'] = 'some_type'
self.db_backend_interface.add_object(self.child_fo)
self.other_fw = create_test_firmware()
self.other_fw.uid = '1234abcd_123'
self.db_backend_interface.add_object(self.other_fw)
class TestAcceptanceDownloadFile(TestAcceptanceBase):
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend = BackEndDbInterface(config=self.config)
time.sleep(10) # wait for systems to start
def tearDown(self):
self.db_backend.shutdown()
self._stop_backend()
super().tearDown()
def _show_analysis_page(self, fw):
rv = self.test_client.get('/analysis/{}'.format(fw.uid))
self.assertIn(fw.uid.encode(), rv.data)
self.assertIn(b'test_router', rv.data)
self.assertIn(b'Router', rv.data)
self.assertIn(b'test_vendor', rv.data)
def _start_binary_download(self, fw):
rv = self.test_client.get('/download/{}'.format(fw.uid))
self.assertIn(fw.binary, rv.data, 'firmware download unsuccessful')
self.assertIn('attachment; filename=test.zip',
rv.headers['Content-Disposition'])
def _start_tar_download(self, fw):
rv = self.test_client.get('/tar-download/{}'.format(fw.uid))
self.assertNotIn(
fw.binary, rv.data,
'tar download yielded original file instead of tar archive')
tar_file = tarfile.open(fileobj=io.BytesIO(rv.data))
contents = ', '.join(tar_file.getnames())
self.assertIn('testfile1', contents,
'test files could not be found in tar download')
def test_firmware_download(self):
test_fw = create_test_firmware()
test_fw.processed_analysis.pop('dummy')
test_fw.uid = test_fw.get_uid()
self.db_backend.add_firmware(test_fw)
self.assertIsNotNone(self.db_backend.firmwares.find_one(test_fw.uid))
self._show_analysis_page(test_fw)
self._start_binary_download(test_fw)
self._start_tar_download(test_fw)
class TestAcceptanceBaseFullStart(TestAcceptanceBase):
NUMBER_OF_FILES_TO_ANALYZE = 4
NUMBER_OF_PLUGINS = 2
def setUp(self):
super().setUp()
self.analysis_finished_event = Event()
self.compare_finished_event = Event()
self.elements_finished_analyzing = Value('i', 0)
self.db_backend_service = BackEndDbInterface(config=self.config)
self._start_backend(post_analysis=self._analysis_callback, compare_callback=self._compare_callback)
time.sleep(2) # wait for systems to start
def tearDown(self):
self._stop_backend()
self.db_backend_service.shutdown()
super().tearDown()
def _analysis_callback(self, fo):
self.db_backend_service.add_object(fo)
self.elements_finished_analyzing.value += 1
if self.elements_finished_analyzing.value == self.NUMBER_OF_FILES_TO_ANALYZE * self.NUMBER_OF_PLUGINS:
self.analysis_finished_event.set()
def _compare_callback(self):
self.compare_finished_event.set()
def upload_test_firmware(self, test_fw):
testfile_path = Path(get_test_data_dir()) / test_fw.path
with open(str(testfile_path), 'rb') as fp:
data = {
'file': (fp, test_fw.file_name),
'device_name': test_fw.name,
'device_part': 'test_part',
'device_class': 'test_class',
'version': '1.0',
'vendor': 'test_vendor',
'release_date': '1970-01-01',
'tags': '',
'analysis_systems': []
}
rv = self.test_client.post('/upload', content_type='multipart/form-data', data=data, follow_redirects=True)
self.assertIn(b'Upload Successful', rv.data, 'upload not successful')
self.assertIn(test_fw.uid.encode(), rv.data, 'uid not found on upload success page')
def setUp(self):
self.admin_interface = AdminDbInterface(config=self.config)
self.db_backend_interface = BackEndDbInterface(config=self.config)
copyfile(TEST_FIRMWARE_ORIGINAL, TEST_FIRMWARE_COPY)
self.test_firmware = create_test_firmware(
bin_path='container/test_copy.zip')
self.uid = self.test_firmware.uid
self.test_firmware.virtual_file_path = {
self.uid: ['|{}|'.format(self.test_firmware.uid)]
}
copyfile(TEST_FILE_ORIGINAL, TEST_FILE_COPY)
self.child_fo = create_test_file_object(TEST_FILE_COPY)
self.child_fo.virtual_file_path = {
self.uid:
['|{}|/folder/{}'.format(self.uid, self.child_fo.file_name)]
}
self.test_firmware.files_included = [self.child_fo.uid]
self.child_uid = self.child_fo.uid
def setUp(self):
self.admin_interface = AdminDbInterface(config=self.config)
self.db_backend_interface = BackEndDbInterface(config=self.config)
copyfile(test_firmware_original, test_firmware_copy)
self.test_firmware = create_test_firmware(
bin_path='container/test_copy.zip')
self.uid = self.test_firmware.get_uid()
self.test_firmware.virtual_file_path = {
self.uid: ['|{}|'.format(self.test_firmware.get_uid())]
}
copyfile(test_file_original, test_file_copy)
self.child_fo = create_test_file_object(test_file_copy)
self.child_fo.virtual_file_path = {
self.uid:
['|{}|/folder/{}'.format(self.uid, self.child_fo.file_name)]
}
self.test_firmware.files_included = [self.child_fo.get_uid()]
self.child_uid = self.child_fo.get_uid()
class TestBinaryService(unittest.TestCase):
def setUp(self):
self.config = get_config_for_testing()
self.mongo_server = MongoMgr(config=self.config)
self._init_test_data()
self.binary_service = BinaryService(config=self.config)
def _init_test_data(self):
self.backend_db_interface = BackEndDbInterface(config=self.config)
self.backend_db_interface.add_firmware(TEST_FW)
self.backend_db_interface.shutdown()
def tearDown(self):
self.mongo_server.shutdown()
gc.collect()
def test_get_binary_and_file_name(self):
binary, file_name = self.binary_service.get_binary_and_file_name(
TEST_FW.uid)
self.assertEqual(file_name, TEST_FW.file_name, 'file_name not correct')
self.assertEqual(binary, TEST_FW.binary, 'invalid result not correct')
def test_get_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_binary_and_file_name(
'invalid_uid')
self.assertIsNone(binary, 'should be none')
self.assertIsNone(file_name, 'should be none')
def test_get_repacked_binary_and_file_name(self):
tar, file_name = self.binary_service.get_repacked_binary_and_file_name(
TEST_FW.uid)
self.assertEqual(file_name, '{}.tar.gz'.format(TEST_FW.file_name),
'file_name not correct')
file_type = magic.from_buffer(tar, mime=False)
assert 'gzip compressed data' in file_type, 'Result is not an tar.gz file'
def test_get_repacked_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_repacked_binary_and_file_name(
'invalid_uid')
self.assertIsNone(binary, 'should be none')
self.assertIsNone(file_name, 'should be none')
class TestBinaryService:
def setup(self):
self.tmp_dir = TemporaryDirectory()
self.config = get_config_for_testing(temp_dir=self.tmp_dir)
self.mongo_server = MongoMgr(config=self.config)
self._init_test_data()
self.binary_service = BinaryService(config=self.config)
def _init_test_data(self):
self.backend_db_interface = BackEndDbInterface(config=self.config)
self.backend_db_interface.add_firmware(TEST_FW)
store_binary_on_file_system(self.tmp_dir.name, TEST_FW)
self.backend_db_interface.shutdown()
def teardown(self):
self.tmp_dir.cleanup()
self.mongo_server.shutdown()
gc.collect()
def test_get_binary_and_file_name(self):
binary, file_name = self.binary_service.get_binary_and_file_name(TEST_FW.uid)
assert file_name == TEST_FW.file_name, 'file_name not correct'
assert binary == TEST_FW.binary, 'invalid result not correct'
def test_get_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_binary_and_file_name('invalid_uid')
assert binary is None, 'should be none'
assert file_name is None, 'should be none'
def test_get_repacked_binary_and_file_name(self):
tar, file_name = self.binary_service.get_repacked_binary_and_file_name(TEST_FW.uid)
assert file_name == '{}.tar.gz'.format(TEST_FW.file_name), 'file_name not correct'
file_type = magic.from_buffer(tar, mime=False)
assert 'gzip compressed data' in file_type, 'Result is not an tar.gz file'
def test_get_repacked_binary_and_file_name_invalid_uid(self):
binary, file_name = self.binary_service.get_repacked_binary_and_file_name('invalid_uid')
assert binary is None, 'should be none'
assert file_name is None, 'should be none'
def setUp(self):
self.db_interface = MongoInterfaceCommon(config=self._config)
self.db_interface_backend = BackEndDbInterface(config=self._config)
self.test_firmware = create_test_firmware()
self.test_yara_match = {
'rule': 'OpenSSH',
'tags': [],
'namespace': 'default',
'strings': [(0, '$a', b'OpenSSH')],
'meta': {
'description': 'SSH library',
'website': 'http://www.openssh.com',
'open_source': True,
'software_name': 'OpenSSH'
},
'matches': True
}
self.test_fo = create_test_file_object()
def __init__(self, config=None, post_analysis=None, db_interface=None):
self.config = config
self.load_plugins()
self.stop_condition = Value('i', 0)
self.process_queue = Queue()
self.tag_queue = Queue()
self.db_backend_service = db_interface if db_interface else BackEndDbInterface(config=config)
self.post_analysis = self.db_backend_service.add_object if post_analysis is None else post_analysis
self.start_scheduling_process()
self.start_result_collector()
logging.info('Analysis System online...')
logging.info('Plugins available: {}'.format(self.get_list_of_available_plugins()))
class TestStatisticWithDb(TestStatisticBase):
def setUp(self):
super().setUp()
self.db_backend_interface = BackEndDbInterface(config=self.config)
def tearDown(self):
self.db_backend_interface.client.drop_database(
self.config.get('data_storage', 'main_database'))
self.db_backend_interface.shutdown()
super().tearDown()
def test_get_executable_stats(self):
for i, file_str in enumerate([
'ELF 64-bit LSB executable, x86-64, dynamically linked, for GNU/Linux 2.6.32, not stripped',
'ELF 32-bit MSB executable, MIPS, MIPS32 rel2 version 1 (SYSV), statically linked, not stripped',
'ELF 64-bit LSB executable, x86-64, (SYSV), corrupted section header size',
'ELF 64-bit LSB executable, aarch64, dynamically linked, stripped',
'ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, stripped'
]):
fo = create_test_file_object()
fo.processed_analysis['file_type'] = {'full': file_str}
fo.uid = str(i)
self.db_backend_interface.add_file_object(fo)
stats = self.updater.get_executable_stats().get('executable_stats')
expected = [('big endian', 1, 0.25), ('little endian', 3, 0.75),
('stripped', 1, 0.25), ('not stripped', 2, 0.5),
('32-bit', 1, 0.25), ('64-bit', 3, 0.75),
('dynamically linked', 2, 0.5),
('statically linked', 1, 0.25),
('section info missing', 1, 0.25)]
for (expected_label, expected_count,
expected_percentage), (label, count, percentage,
_) in zip(expected, stats):
assert label == expected_label
assert count == expected_count
assert percentage == expected_percentage
def __init__(self, config: Optional[ConfigParser] = None, pre_analysis=None, post_analysis=None, db_interface=None):
self.config = config
self.analysis_plugins = {}
self._load_plugins()
self.stop_condition = Value('i', 0)
self.process_queue = Queue()
self.status = AnalysisStatus()
self.task_scheduler = AnalysisTaskScheduler(self.analysis_plugins)
self.db_backend_service = db_interface if db_interface else BackEndDbInterface(config=config)
self.pre_analysis = pre_analysis if pre_analysis else self.db_backend_service.add_object
self.post_analysis = post_analysis if post_analysis else self.db_backend_service.add_analysis
self._start_runner_process()
self._start_result_collector()
logging.info('Analysis System online...')
logging.info(f'Plugins available: {self._get_list_of_available_plugins()}')
class TestAcceptanceAdvancedSearch(TestAcceptanceBase):
def setUp(self):
super().setUp()
self._start_backend()
self.db_backend_interface = BackEndDbInterface(self.config)
self.parent_fw = create_test_firmware()
self.child_fo = create_test_file_object()
uid = self.parent_fw.uid
self.child_fo.virtual_file_path = {uid: ['|{}|/folder/{}'.format(uid, self.child_fo.file_name)]}
self.db_backend_interface.add_object(self.parent_fw)
self.child_fo.processed_analysis['unpacker'] = {}
self.child_fo.processed_analysis['unpacker']['plugin_used'] = 'test'
self.child_fo.processed_analysis['file_type']['mime'] = 'some_type'
self.db_backend_interface.add_object(self.child_fo)
def tearDown(self):
self.db_backend_interface.shutdown()
self._stop_backend()
super().tearDown()
def test_advanced_search_get(self):
rv = self.test_client.get('/database/advanced_search')
assert b'<h2>Advanced Search</h2>' in rv.data
def test_advanced_search(self):
rv = self.test_client.post('/database/advanced_search', content_type='multipart/form-data',
data={'advanced_search': '{}'}, follow_redirects=True)
assert b'Please enter a valid search request' not in rv.data
assert self.parent_fw.uid.encode() in rv.data
assert self.child_fo.uid.encode() not in rv.data
def test_advanced_search_file_object(self):
rv = self.test_client.post('/database/advanced_search', content_type='multipart/form-data',
data={'advanced_search': json.dumps({'_id': self.child_fo.uid})}, follow_redirects=True)
assert b'Please enter a valid search request' not in rv.data
assert b'<strong>UID:</strong> ' + self.parent_fw.uid.encode() not in rv.data
assert b'<strong>UID:</strong> ' + self.child_fo.uid.encode() in rv.data
def test_advanced_search_only_firmwares(self):
rv = self.test_client.post('/database/advanced_search', content_type='multipart/form-data',
data={'advanced_search': json.dumps({'_id': self.child_fo.uid}), 'only_firmwares': 'True'}, follow_redirects=True)
assert b'Please enter a valid search request' not in rv.data
assert self.child_fo.uid.encode() not in rv.data
assert self.parent_fw.uid.encode() in rv.data
def test_rest_recursive_firmware_search(self):
query = quote(json.dumps({'file_name': self.child_fo.file_name}))
response = self.test_client.get('/rest/firmware?recursive=true&query={}'.format(query)).data
assert b'error_message' not in response
assert self.parent_fw.uid.encode() in response
def __init__(self, config: Optional[ConfigParser] = None, pre_analysis=None, post_analysis=None, db_interface=None):
self.config = config
self.analysis_plugins = {}
self.load_plugins()
self.stop_condition = Value('i', 0)
self.process_queue = Queue()
self.tag_queue = Queue()
self.manager = Manager()
self.currently_running = self.manager.dict()
self.recently_finished = self.manager.dict()
self.currently_running_lock = self.manager.Lock() # pylint: disable=no-member
self.db_backend_service = db_interface if db_interface else BackEndDbInterface(config=config)
self.pre_analysis = pre_analysis if pre_analysis else self.db_backend_service.add_object
self.post_analysis = post_analysis if post_analysis else self.db_backend_service.add_analysis
self.start_scheduling_process()
self.start_result_collector()
logging.info('Analysis System online...')
logging.info('Plugins available: {}'.format(self.get_list_of_available_plugins()))
class TestRestMissingAnalyses(RestTestBase):
def setUp(self):
super().setUp()
self.db_backend = BackEndDbInterface(config=self.config)
def tearDown(self):
self.db_backend.shutdown()
super().tearDown()
def test_rest_get_missing_files(self):
test_fw = create_test_firmware()
missing_uid = 'uid1234'
test_fw.files_included.add(missing_uid)
self.db_backend.add_firmware(test_fw)
response = json.loads(self.test_client.get('/rest/missing', follow_redirects=True).data.decode())
assert 'missing_files' in response
assert test_fw.uid in response['missing_files']
assert missing_uid in response['missing_files'][test_fw.uid]
assert response['missing_analyses'] == {}
def test_rest_get_missing_analyses(self):
test_fw = create_test_firmware()
test_fo = create_test_file_object()
test_fw.files_included.add(test_fo.uid)
test_fo.virtual_file_path = {test_fw.uid: ['|foo|bar|']}
test_fw.processed_analysis['foobar'] = {'foo': 'bar'}
# test_fo is missing this analysis but is in files_included -> should count as missing analysis
self.db_backend.add_firmware(test_fw)
self.db_backend.add_file_object(test_fo)
response = json.loads(self.test_client.get('/rest/missing', follow_redirects=True).data.decode())
assert 'missing_analyses' in response
assert test_fw.uid in response['missing_analyses']
assert test_fo.uid in response['missing_analyses'][test_fw.uid]
assert response['missing_files'] == {}
def __init__(self,
config: Optional[ConfigParser] = None,
pre_analysis=None,
post_analysis=None,
db_interface=None):
self.config = config
self.analysis_plugins = {}
self.load_plugins()
self.stop_condition = Value('i', 0)
self.process_queue = Queue()
self.tag_queue = Queue()
# db_backend_service 创建与数据db之间的接口处理对象
self.db_backend_service = db_interface if db_interface else BackEndDbInterface(
config=config)
# pre_analysis 对应db_backend_service.add_ojbect()方法
self.pre_analysis = pre_analysis if pre_analysis else self.db_backend_service.add_object
# post_analysis 对应 db_backend_service.add_analysis() 方法
self.post_analysis = post_analysis if post_analysis else self.db_backend_service.add_analysis
#
self.start_scheduling_process()
self.start_result_collector()
logging.info('Analysis System online...')
logging.info('Plugins available: {}'.format(
self.get_list_of_available_plugins()))
