def test_process_next_analysis_unknown_plugin(self):
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['unknown_plugin']
with mock_spy(self.sched, '_start_or_skip_analysis') as spy:
self.sched.process_next_analysis(test_fw)
assert not spy.was_called(), 'unknown plugin should simply be skipped'
def test_add_tag(input_data, expected_count):
test_object = Firmware()
for item in input_data:
test_object.set_tag(item)
for item in input_data:
assert item in test_object.tags
assert len(test_object.tags.keys()) == expected_count
def test_set_binary():
binary = get_binary_from_file('{}/get_files_test/testfile1'.format(
get_test_data_dir()))
md5 = 'e802ca22f6cd2d9357cf3da1d191879e'
firmware = Firmware()
firmware.set_binary(binary)
assert firmware.md5 == md5
def test_create_firmware_from_file(self):
test_object = Firmware()
test_object.create_from_file('{}/test_data_file.bin'.format(get_test_data_dir()))
self.assertEqual(test_object.device_name, None, 'correct device name')
self.assertEqual(test_object.size, 19, 'correct size')
self.assertEqual(test_object.binary, b'test string in file', 'correct binary data')
self.assertEqual(test_object.sha256, '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8', 'correct sha256')
self.assertEqual(test_object.file_name, 'test_data_file.bin', 'correct file name')
def test_create_firmware_from_file():
test_object = Firmware()
test_object.create_from_file('{}/test_data_file.bin'.format(get_test_data_dir()))
assert test_object.device_name is None
assert test_object.size == 19
assert test_object.binary == b'test string in file'
assert test_object.sha256 == '268d870ffa2b21784e4dc955d8e8b8eb5f3bcddd6720a1e6d31d2cf84bd1bff8'
assert test_object.file_name == 'test_data_file.bin'
def test_add_firmware_to_current_analyses(self):
self.scheduler.currently_running = {}
fw = Firmware(binary=b'foo')
fw.files_included = ['foo', 'bar']
self.scheduler._add_to_current_analyses(fw)
assert fw.uid in self.scheduler.currently_running
assert self.scheduler.currently_running[fw.uid]['file_list'] == ['foo', 'bar']
assert self.scheduler.currently_running[fw.uid]['analyzed_files_count'] == 0
assert self.scheduler.currently_running[fw.uid]['total_files_count'] == 2
def test_skip_analysis_because_whitelist(self):
self.sched.config.set('dummy_plugin_for_testing_only', 'mime_whitelist', 'foo, bar')
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['file_hashes']
test_fw.processed_analysis['file_type'] = {'mime': 'text/plain'}
self.sched._start_or_skip_analysis('dummy_plugin_for_testing_only', test_fw)
test_fw = self.tmp_queue.get(timeout=10)
assert 'dummy_plugin_for_testing_only' in test_fw.processed_analysis
assert 'skipped' in test_fw.processed_analysis['dummy_plugin_for_testing_only']
def test_whole_run_analysis_selected(self):
test_fw = Firmware(file_path=os.path.join(get_test_data_dir(), 'get_files_test/testfile1'))
test_fw.scheduled_analysis = ['dummy_plugin_for_testing_only']
self.sched.add_task(test_fw)
test_fw = self.tmp_queue.get(timeout=10)
self.assertEqual(len(test_fw.processed_analysis), 3, 'analysis not done')
self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['1'], 'first result', 'result not correct')
self.assertEqual(test_fw.processed_analysis['dummy_plugin_for_testing_only']['summary'], ['first result', 'second result'])
self.assertIn('file_hashes', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
self.assertIn('file_type', test_fw.processed_analysis.keys(), 'Mandatory plug-in not executed')
def test_add_firmware_to_current_analyses(self):
self.status.currently_running = {}
fw = Firmware(binary=b'foo')
fw.files_included = ['foo', 'bar']
self.status.add_to_current_analyses(fw)
assert fw.uid in self.status.currently_running
result = self.status.currently_running[fw.uid]
assert result['files_to_unpack'] == ['foo', 'bar']
assert result['files_to_analyze'] == [fw.uid]
assert result['unpacked_files_count'] == 1
assert result['analyzed_files_count'] == 0
assert result['total_files_count'] == 3
def test_get_hid(input_data, expected_output):
test_fw = Firmware(binary=b'foo')
test_fw.device_name = 'test_device'
test_fw.vendor = 'foo'
test_fw.version = '1.0'
test_fw.set_part_name(input_data)
assert test_fw.get_hid() == expected_output
def test_reschedule_failed_analysis_task(self):
task = Firmware(binary='foo')
error_message = 'There was an exception'
task.analysis_exception = ('foo', error_message)
task.scheduled_analysis = ['no_deps', 'bar']
task.processed_analysis['foo'] = {'error': 1}
self._add_plugins()
self.scheduler._reschedule_failed_analysis_task(task)
assert 'foo' in task.processed_analysis
assert task.processed_analysis['foo'] == {'failed': error_message}
assert 'bar' not in task.scheduled_analysis
assert 'bar' in task.processed_analysis
assert task.processed_analysis['bar'] == {'failed': 'Analysis of dependency foo failed'}
assert 'no_deps' in task.scheduled_analysis
def test_run_analysis_with_tag(self):
test_fw = Firmware(file_path='{}/container/with_key.7z'.format(get_test_data_dir()))
test_fw.release_date = '2017-01-01'
test_fw.scheduled_analysis = ['crypto_material']
self._unpack_scheduler.add_task(test_fw)
assert self.analysis_finished_event.wait(timeout=20)
processed_fo = self.backend_interface.get_object(self.uid_of_key_file, analysis_filter=['crypto_material'])
assert processed_fo.processed_analysis['crypto_material']['tags'], 'no tags set in analysis'
processed_fw = self.backend_interface.get_object(test_fw.uid, analysis_filter=['crypto_material'])
assert processed_fw.analysis_tags, 'tags not propagated properly'
assert processed_fw.analysis_tags['crypto_material']['private_key_inside']
def test_unpack_a_container_including_another_container(self):
self._start_scheduler()
test_fw = Firmware(file_path='{}/container/test_zip.tar.gz'.format(
get_test_data_dir()))
self.sched.add_task(test_fw)
outer_container = self.tmp_queue.get(timeout=5)
self.assertEqual(len(outer_container.files_included), 2,
'not all childs of root found')
self.assertIn(
'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227',
outer_container.files_included,
'included container not extracted. Unpacker tar.gz modul broken?')
included_files = [self.tmp_queue.get(timeout=5)]
included_files.append(self.tmp_queue.get(timeout=5))
for item in included_files:
if item.get_uid(
) == 'ab4153d747f530f9bc3a4b71907386f50472ea5ae975c61c0bacd918f1388d4b_227':
self.assertEqual(
len(item.files_included), 1,
'number of files in included container not correct')
else:
self.assertEqual(
item.get_uid(),
'faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28',
'none container file not rescheduled')
def convert_analysis_task_to_fw_obj(analysis_task: dict,
base_fw: Optional[Firmware] = None
) -> Firmware:
'''
Convert an analysis task to a firmware object.
:param analysis_task: The analysis task data.
:param base_fw: The existing `Firmware` object in case of analysis update.
:return: A `Firmware` object based on the analysis task data.
'''
fw = base_fw or Firmware()
fw.scheduled_analysis = analysis_task['requested_analysis_systems']
if 'binary' in analysis_task.keys():
fw.set_binary(analysis_task['binary'])
fw.file_name = analysis_task['file_name']
else:
if 'file_name' in analysis_task.keys():
fw.file_name = analysis_task['file_name']
fw.uid = analysis_task['uid']
fw.device_name = analysis_task['device_name']
fw.set_part_name(analysis_task['device_part'])
fw.version = analysis_task['version']
fw.device_class = analysis_task['device_class']
fw.vendor = analysis_task['vendor']
fw.release_date = analysis_task['release_date']
for tag in _get_tag_list(analysis_task['tags']):
fw.set_tag(tag)
return fw
def test_unpack_and_analyse(self):
test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
processed_container = self._tmp_queue.get(timeout=10)
self.assertGreaterEqual(len(processed_container.processed_analysis), 3, 'at least one analysis not done')
def test_schedule_firmware_init_no_analysis_selected(self):
self.sched.shutdown()
self.sched.process_queue = Queue()
test_fw = Firmware(binary=b'test')
self.sched.add_task(test_fw)
test_fw = self.sched.process_queue.get(timeout=5)
self.assertEqual(len(test_fw.scheduled_analysis), len(MANDATORY_PLUGINS), 'Mandatory Plugins not selected')
for item in MANDATORY_PLUGINS:
self.assertIn(item, test_fw.scheduled_analysis)
def test_unpack_only(self):
test_fw = Firmware(file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
processed_container = self._tmp_queue.get(timeout=5)
self.assertEqual(len(processed_container.files_included), 3, 'not all included files found')
self.assertIn('faa11db49f32a90b51dfc3f0254f9fd7a7b46d0b570abd47e1943b86d554447a_28', processed_container.files_included, 'certain file missing after unpacking')
def _init_current_analysis(fw_object: Firmware):
return {
'files_to_unpack': list(fw_object.files_included),
'files_to_analyze': [fw_object.uid],
'start_time': time(),
'unpacked_files_count': 1,
'analyzed_files_count': 0,
'total_files_count': 1 + len(fw_object.files_included),
'hid': fw_object.get_hid(),
}
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/container/test.7z'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=10)
compare_id = unify_string_list(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertFalse(isinstance(result, str),
'compare result should exist')
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['exclusive_files'],
self._expected_result()['File_Coverage']['exclusive_files'])
def test_unpack_analyse_and_compare(self):
test_fw_1 = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
test_fw_1.release_date = '2017-01-01'
test_fw_2 = Firmware(
file_path='{}/regression_one'.format(get_test_data_dir()))
test_fw_2.release_date = '2017-01-01'
self._unpack_scheduler.add_task(test_fw_1)
self._unpack_scheduler.add_task(test_fw_2)
self.analysis_finished_event.wait(timeout=20)
compare_id = normalize_compare_id(';'.join(
[fw.uid for fw in [test_fw_1, test_fw_2]]))
self.assertIsNone(
self._compare_scheduler.add_task((compare_id, False)),
'adding compare task creates error')
self.compare_finished_event.wait(timeout=10)
with ConnectTo(CompareDbInterface, self._config) as sc:
result = sc.get_compare_result(compare_id)
self.assertEqual(result['plugins']['Software'],
self._expected_result()['Software'])
self.assertCountEqual(
result['plugins']['File_Coverage']['files_in_common'],
self._expected_result()['File_Coverage']['files_in_common'])
def test_unpack_and_analyse(self):
test_fw = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
for _ in range(
12
): # magic to make tests pass. after switch of post analysis, each analysis gets the post call
processed_container = self._tmp_queue.get(timeout=10)
self.assertGreaterEqual(len(processed_container.processed_analysis), 3,
'at least one analysis not done')
def test_unpack_and_analyse(self):
test_fw = Firmware(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
self._unpack_scheduler.add_task(test_fw)
for _ in range(
4 * 2
): # container with 3 included files times 2 mandatory plugins run
processed_container = self._tmp_queue.get(timeout=10)
self.assertGreaterEqual(len(processed_container.processed_analysis), 3,
'at least one analysis not done')
def _convert_to_firmware(self, entry: dict, analysis_filter: List[str] = None) -> Firmware:
firmware = Firmware()
firmware.uid = entry['_id']
firmware.size = entry['size']
firmware.file_name = entry['file_name']
firmware.device_name = entry['device_name']
firmware.device_class = entry['device_class']
firmware.release_date = convert_time_to_str(entry['release_date'])
firmware.vendor = entry['vendor']
firmware.version = entry['version']
firmware.processed_analysis = self.retrieve_analysis(entry['processed_analysis'], analysis_filter=analysis_filter)
firmware.files_included = set(entry['files_included'])
firmware.virtual_file_path = entry['virtual_file_path']
firmware.tags = entry['tags'] if 'tags' in entry else dict()
firmware.analysis_tags = self._collect_analysis_tags_from_children(firmware.uid)
try: # for backwards compatibility
firmware.set_part_name(entry['device_part'])
except KeyError:
firmware.set_part_name('complete')
if 'comments' in entry: # for backwards compatibility
firmware.comments = entry['comments']
return firmware
def test_get_hid(input_data, expected_output):
test_fw = Firmware(binary=b'foo')
test_fw.set_device_name('test_device')
test_fw.set_vendor('foo')
test_fw.set_firmware_version('1.0')
test_fw.set_part_name(input_data)
assert test_fw.get_hid() == expected_output
def test_repr_and_str():
test_fw = Firmware(scheduled_analysis=['test'])
assert 'None None v. None' in test_fw.__str__()
assert 'test' in test_fw.__str__()
assert test_fw.__str__() == test_fw.__repr__()
def _convert_to_firmware(self, entry, analysis_filter=None):
firmware = Firmware()
firmware.uid = entry['_id']
firmware.size = entry['size']
firmware.set_name(entry['file_name'])
firmware.set_device_name(entry['device_name'])
firmware.set_device_class(entry['device_class'])
firmware.set_release_date(convert_time_to_str(entry['release_date']))
firmware.set_vendor(entry['vendor'])
firmware.set_firmware_version(entry['version'])
firmware.processed_analysis = self.retrieve_analysis(entry['processed_analysis'], analysis_filter=analysis_filter)
firmware.files_included = set(entry['files_included'])
firmware.virtual_file_path = entry['virtual_file_path']
firmware.tags = entry['tags'] if 'tags' in entry else dict()
firmware.analysis_tags = entry['analysis_tags'] if 'analysis_tags' in entry else dict()
if 'comments' in entry: # for backwards compatibility
firmware.comments = entry['comments']
return firmware
def test_create_firmware_container_raw():
test_object = Firmware()
assert test_object.size is None
assert test_object.binary is None
def test_remove_tag(tag_set, remove_items, expected_count):
test_fw = Firmware()
test_fw.tags = tag_set
for item in remove_items:
test_fw.remove_tag(item)
assert len(test_fw.tags.keys()) == expected_count
def test_set_part_name(input_data, expected_output):
test_object = Firmware()
test_object.set_part_name(input_data)
assert test_object.part == expected_output
def create_test_firmware(device_class='Router',
device_name='test_router',
vendor='test_vendor',
bin_path='container/test.zip',
all_files_included_set=False,
version='0.1'):
fw = Firmware(file_path=os.path.join(get_test_data_dir(), bin_path))
fw.set_device_class(device_class)
fw.set_device_name(device_name)
fw.set_vendor(vendor)
fw.set_release_date('1970-01-01')
fw.version = version
processed_analysis = {
'dummy': {
'summary': ['sum a', 'fw exclusive sum a'],
'content': 'abcd'
},
'unpacker': {
'plugin_used': 'used_unpack_plugin'
},
'file_type': {
'mime': 'test_type',
'full': 'Not a PE file',
'summary': ['a summary']
}
}
fw.processed_analysis.update(processed_analysis)
if all_files_included_set:
fw.list_of_all_included_files = list(fw.files_included)
fw.list_of_all_included_files.append(fw.uid)
return fw
