def test_detect_unpack_loss_no_data_lost(self):
container = FileObject(binary=512 * 'ABCDEFGH')
container.processed_analysis['unpacker'] = {'summary': []}
included_file = FileObject(binary=512 * 'ABCDEFGH')
self.unpacker._detect_unpack_loss(container, [included_file])
self.assertIn('no data lost', container.processed_analysis['unpacker']['summary'])
self.assertNotIn('data loss', container.processed_analysis['unpacker'])
def test_process_object(self):
test_file = FileObject(
file_path=os.path.join(TEST_DATA_DIR, 'passwd_test'))
processed_object = self.analysis_plugin.process_object(test_file)
results = processed_object.processed_analysis[self.PLUGIN_NAME]
self.assertEqual(len(results), 7)
for s in ['vboxadd', 'mongodb', 'clamav', 'pulse', 'johndoe', 'max']:
self.assertIn(s, results)
self.assertIn(s, results['summary'])
self.assertIn('password-hash', results['max'])
self.assertIn('password', results['max'])
self.assertEqual(results['max']['password'], 'dragon')
self.assertIn('password-hash', results['johndoe'])
self.assertIn('password', results['johndoe'])
self.assertEqual(results['johndoe']['password'], '123456')
test_file = FileObject(
file_path=os.path.join(TEST_DATA_DIR, 'passwd.bin'))
processed_object = self.analysis_plugin.process_object(test_file)
results = processed_object.processed_analysis[self.PLUGIN_NAME]
self.assertEqual(len(results), 3)
for s in ['johndoe', 'max']:
self.assertIn(s, results)
self.assertIn(s, results['summary'])
self.assertIn('password-hash', results['johndoe'])
self.assertIn('password', results['johndoe'])
self.assertEqual(results['johndoe']['password'], '123456')
self.assertIn('password-hash', results['max'])
self.assertIn('password', results['max'])
self.assertEqual(results['max']['password'], 'dragon')
def test_process_objects_kernel_image(self):
for valid_image in glob.glob(str(TEST_DATA_DIR / 'synthetic/*.image')):
test_file = FileObject(file_path=str(valid_image))
test_file.processed_analysis['file_type'] = dict(
mime='application/octet-stream')
test_file.processed_analysis['software_components'] = dict(
summary=['Linux Kernel'])
self.analysis_plugin.process_object(test_file)
assert test_file.processed_analysis[
self.PLUGIN_NAME]['is_kernel_config']
assert len(test_file.processed_analysis[self.PLUGIN_NAME]
['kernel_config']) > 0
for bad_image in glob.glob(
str(TEST_DATA_DIR / 'random_invalid/*.image')):
test_file = FileObject(file_path=str(bad_image))
test_file.processed_analysis['file_type'] = dict(
mime='application/octet-stream')
test_file.processed_analysis['software_components'] = dict(
summary=['Linux Kernel'])
self.analysis_plugin.process_object(test_file)
assert 'is_kernel_config' not in test_file.processed_analysis[
self.PLUGIN_NAME]
assert 'kernel_config' not in test_file.processed_analysis[
self.PLUGIN_NAME]
def setUpClass(cls):
test_init_dir = os.path.join(get_dir_of_file(__file__), 'data')
test_files = {
'systemd': 'etc/systemd/system/foobar',
'inittab': 'etc/inittab',
'rclocal': 'etc/rc.local',
'upstart': 'etc/init/baz.conf',
'runit': 'etc/service/lighttpd/run',
'runit_symlink': 'etc/service/example/run',
'runit_origin': 'etc/sv/example/run',
'only_comments': 'etc/inittab.invalid',
'initd': 'etc/init.d/skeleton',
'README': 'etc/init.d/README',
'initscript': 'etc/initscript'
}
for test_file, path in test_files.items():
test_fo = FileObject(file_path=os.path.join(test_init_dir, path))
setattr(cls, 'test_file_{}'.format(test_file), test_fo)
test_fo.processed_analysis['file_type'] = {'mime': 'text/plain'}
test_fo.root_uid = test_fo.uid
test_fo.virtual_file_path = {test_fo.get_root_uid(): [path]}
cls.test_file_not_text = FileObject(
file_path='{}etc/systemd/system/foobar'.format(test_init_dir))
cls.test_file_not_text.processed_analysis['file_type'] = {
'mime': 'application/zip'
}
def test_detect_unpack_loss_data_lost(self):
container = FileObject(binary=512 * 'ABCDEFGH')
container.processed_analysis['unpacker'] = {'summary': []}
included_file = FileObject(binary=256 * 'ABCDEFGH')
self.unpacker._detect_unpack_loss(container, [included_file])
self.assertIn('data lost', container.processed_analysis['unpacker']['summary'])
self.assertEqual(container.processed_analysis['unpacker']['size packed -> unpacked'], '3.75 KiB -> 2.00 KiB')
def test_get_included_files_uids(self):
test_parent = FileObject(binary=b'parent_file')
test_child = FileObject(binary=b'1st child')
test_child2 = FileObject(binary=b'2nd child')
test_parent.add_included_file(test_child)
test_parent.add_included_file(test_child2)
self.assertEqual(len(test_parent.get_included_files_uids()), 2, 'number of uids not correct')
self.assertIn(test_child.get_uid(), test_parent.get_included_files_uids(), 'uid of first file not found')
self.assertIn(test_child2.get_uid(), test_parent.get_included_files_uids(), 'uid of second file not found')
def test_get_included_files(self):
test_parent = FileObject(binary=b'parent_file')
test_child = FileObject(binary=b'1st child')
test_child2 = FileObject(binary=b'2nd child')
test_parent.add_included_file(test_child)
test_parent.add_included_file(test_child2)
assert len(test_parent.files_included) == 2, 'number of uids not correct'
assert test_child.uid in test_parent.files_included, 'uid of first file not found'
assert test_child2.uid in test_parent.files_included, 'uid of second file not found'
def test_object_processing_one_child(self):
root_object = FileObject(binary=b'root_file')
child_object = FileObject(binary=b'first_child_object')
root_object.add_included_file(child_object)
self.base_plugin.in_queue.put(root_object)
processed_object = self.base_plugin.out_queue.get()
self.assertEqual(processed_object.uid, root_object.uid, 'uid changed')
self.assertTrue(child_object.uid in root_object.files_included,
'child object not in processed file')
def test_add_included_file(self):
parent = FileObject(binary=b'parent_file')
parent.scheduled_analysis = ['test']
child = FileObject(binary=b'child')
parent.add_included_file(child)
assert len(parent.files_included) == 1, 'number of included files not correct'
assert child.uid in parent.files_included, 'child uid not stored correctly'
assert parent.uid in child.parents, 'parent not added to child'
assert child.depth == parent.depth + 1, 'child depth not updated'
assert child.scheduled_analysis == ['test'], 'child did not get scheduled analysis list of parent'
def test_add_included_file(self):
parent = FileObject(binary=b'parent_file')
parent.scheduled_analysis = ['test']
child = FileObject(binary=b'child')
parent.add_included_file(child)
self.assertEqual(len(parent.files_included), 1, 'number of included files not correct')
self.assertIn(child.get_uid(), parent.files_included, 'child uid not stored correctly')
self.assertIn(parent.get_uid(), child.parents, 'parent not added to child')
self.assertEqual(child.depth, parent.depth + 1, 'child depth not updated')
self.assertEqual(child.scheduled_analysis, ['test'], 'child did not get scheduled analysis list of parent')
def test_object_processing_one_child(self):
root_object = FileObject(binary=b'root_file')
child_object = FileObject(binary=b'first_child_object')
root_object.add_included_file(child_object)
self.pBase.in_queue.put(root_object)
processed_object = self.pBase.out_queue.get()
self.assertEqual(processed_object.get_uid(), root_object.get_uid(),
'uid changed')
self.assertTrue(
child_object.get_uid() in root_object.get_included_files_uids(),
'child object not in processed file')
def test_process_object(self):
test_object = FileObject()
test_object.processed_analysis['file_type'] = {'mime': 'linux/device-tree'}
test_object.file_path = str(TEST_FILE)
result = self.analysis_plugin.process_object(test_object)
assert result.processed_analysis[self.PLUGIN_NAME]['summary'] == ['device tree found']
def test_build_bap_command(self):
self.analysis_plugin.docker = True
fo = FileObject(file_path='/foo')
assert self.analysis_plugin._build_bap_command(
fo
) == 'timeout --signal=SIGKILL {}m docker run --rm -v {}:/tmp/input {} bap /tmp/input --pass=cwe-checker --cwe-checker-config=/home/bap/cwe_checker/src/config.json'.format(
BAP_TIMEOUT, fo.file_path, DOCKER_IMAGE)
def test_process_object_shadow_file(self):
test_file = FileObject(file_path=str(TEST_DATA_DIR / 'passwd_test'))
processed_object = self.analysis_plugin.process_object(test_file)
results = processed_object.processed_analysis[self.PLUGIN_NAME]
self.assertEqual(len(results), 10)
for item in ['vboxadd:unix', 'mongodb:unix', 'clamav:unix', 'pulse:unix', 'johndoe:unix', 'max:unix', 'test:mosquitto']:
self.assertIn(item, results)
self.assertIn(item, results['summary'])
self.assertIn('type', results['max:unix'])
self.assertIn('password-hash', results['max:unix'])
self.assertIn('password', results['max:unix'])
self.assertEqual(results['max:unix']['type'], 'unix')
self.assertEqual(results['max:unix']['password'], 'dragon')
self.assertIn('type', results['johndoe:unix'])
self.assertIn('password-hash', results['johndoe:unix'])
self.assertIn('password', results['johndoe:unix'])
self.assertEqual(results['johndoe:unix']['type'], 'unix')
self.assertEqual(results['johndoe:unix']['password'], '123456')
self.assertEqual(results['tags']['johndoe_123456']['value'], 'Password: johndoe:123456')
self.assertIn('type', results['test:mosquitto'])
self.assertIn('password-hash', results['test:mosquitto'])
self.assertIn('password', results['test:mosquitto'])
self.assertEqual(results['test:mosquitto']['type'], 'mosquitto')
self.assertEqual(results['test:mosquitto']['password'], '123456')
self.assertEqual(results['tags']['test_123456']['value'], 'Password: test:123456')
def test_unpacking_depth_reached(self):
test_file = FileObject(file_path=self.test_file_path)
test_file.depth = 10
self.unpacker.unpack(test_file)
assert 'unpacker' in test_file.processed_analysis
assert 'maximum unpacking depth was reached' in test_file.processed_analysis[
'unpacker']['info']
def test_get_root_uid(self):
fo = FileObject(binary=b'file_object')
fo.virtual_file_path = {
'root_uid_1': 'virtual_file_path',
'root_uid_2': 'virtual_file_path'
}
assert fo.get_root_uid() in ['root_uid_1', 'root_uid_2']
def test_process_object(self):
test_file = FileObject(
file_path=os.path.join(TEST_DATA_DIR, 'yara_test_file'))
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
self.assertEqual(len(results), 2,
'incorrect number of software components found')
self.assertTrue('MyTestRule' in results, 'test Rule match not found')
self.assertEqual(results['MyTestRule']['meta']['software_name'],
'Test Software',
'incorrect software name from yara meta')
self.assertEqual(results['MyTestRule']['meta']['website'],
'http://www.fkie.fraunhofer.de',
'incorrect website from yara meta')
self.assertEqual(results['MyTestRule']['meta']['description'],
'This is a test rule',
'incorrect description from yara meta')
self.assertTrue(results['MyTestRule']['meta']['open_source'],
'incorrect open-source flag from yara meta')
self.assertTrue((10, '$a', b'MyTestRule 0.1.3.')
in results['MyTestRule']['strings'],
'string not found')
self.assertTrue('0.1.3' in results['MyTestRule']['meta']['version'],
'Version not detected')
self.assertEqual(len(results['MyTestRule']['strings']), 1,
'to much strings found')
self.assertEqual(len(results['summary']), 1,
'Number of summary results not correct')
self.assertIn('Test Software 0.1.3', results['summary'])
def setUp(self):
super().setUp()
self.test_file = FileObject(
file_path=os.path.join(TEST_DATA_DIR, 'DLM-First_1MB.DLM'))
self.test_firmware = self.test_file.file_path
self.firmware_container = XeroxDLM(self.test_firmware)
def test_remove_fully_from_current_analyses(self):
self.scheduler.currently_running = {'parent_uid': ['foo']}
fo = FileObject(binary=b'foo')
fo.parent_firmware_uids = {'parent_uid'}
fo.uid = 'foo'
self.scheduler._remove_from_current_analyses(fo)
assert self.scheduler.currently_running == {}
def test_probably_kernel_config_utf_error(self):
test_file = FileObject(file_path=str(TEST_DATA_DIR /
'random_invalid/a.image'))
test_file.processed_analysis['file_type'] = dict(mime='text/plain')
assert not self.analysis_plugin.probably_kernel_config(
test_file.binary)
def test_plugin(self):
test_object = FileObject()
self.analysis_plugin.process_object(test_object)
self.assertEqual(
test_object.processed_analysis[self.PLUGIN_NAME]
['analysis_result_a'], 'hello world')
def test_get_virtual_file_path(self):
fo = FileObject(binary=b'file_object')
assert fo.uid in fo.get_virtual_file_paths().keys(), 'not correct if path _ name not set'
fo.file_name = 'the_file_name.txt'
assert fo.get_virtual_file_paths()[fo.uid][0] == fo.uid, 'not correct if path not set'
fo.virtual_file_path = {fo.uid: '/foo/bar/the_file_name.txt'}
assert fo.get_virtual_file_paths()[fo.uid] == '/foo/bar/the_file_name.txt', 'not correct if path set'
def create_test_file_object(bin_path='get_files_test/testfile1'):
fo = FileObject(file_path=os.path.join(get_test_data_dir(), bin_path))
processed_analysis = {'dummy': {'summary': [
'sum a', 'file exclusive sum b'], 'content': 'file abcd'}, 'file_type': {'full': 'Not a PE file'}}
fo.processed_analysis.update(processed_analysis)
fo.virtual_file_path = fo.get_virtual_file_paths()
return fo
def test_entropy_graph(self):
test_file = FileObject(
file_path='{}/container/test.zip'.format(get_test_data_dir()))
processed_file = self.analysis_plugin.process_object(test_file)
results = processed_file.processed_analysis[self.PLUGIN_NAME]
self.assertGreater(len(results['entropy_analysis_graph']), 0,
'no binwalk entropy graph found')
def test_get_virtual_file_path(self):
fo = FileObject(binary=b'file_object')
self.assertIn(fo.get_uid(), fo.get_virtual_file_paths().keys(), 'not correct if path _ name not set')
fo.set_name('the_file_name.txt')
self.assertEqual(fo.get_virtual_file_paths()[fo.get_uid()][0], fo.get_uid(), 'not correct if path not set')
fo.virtual_file_path = {fo.get_uid(): '/foo/bar/the_file_name.txt'}
self.assertEqual(fo.get_virtual_file_paths()[fo.get_uid()], '/foo/bar/the_file_name.txt', 'not correct if path set')
def setUpClass(self):
test_init_dir = os.path.join(get_dir_of_file(__file__), 'data')
test_files = {
'systemd': 'etc/systemd/system/foobar',
'inittab': 'etc/inittab',
'rclocal': 'etc/rc.local',
'upstart': 'etc/init/baz.conf',
'runit': 'etc/service/lighttpd/run',
'runit_symlink': 'etc/service/example/run',
'runit_origin': 'etc/sv/example/run',
'only_comments': 'etc/inittab.invalid',
'initd': 'etc/init.d/skeleton',
'README': 'etc/init.d/README',
'initscript': 'etc/initscript'
}
for test_file, path in test_files.items():
exec("self.test_file_" + test_file +
" = FileObject(file_path=os.path.join(test_init_dir, path))")
exec("self.test_file_" + test_file +
".processed_analysis['file_type'] = {'mime': 'text/plain'}")
exec("self.test_file_" + test_file +
".root_uid = self.test_file_" + test_file + ".uid")
exec("self.test_file_" + test_file +
".virtual_file_path = {self.test_file_" + test_file +
".get_root_uid(): [\"" + path + "\"]}")
self.test_file_not_text = FileObject(
file_path="{}etc/systemd/system/foobar".format(test_init_dir))
self.test_file_not_text.processed_analysis['file_type'] = {
'mime': 'application/zip'
}
def test_probably_kernel_config_false(self):
test_file = FileObject(file_path=str(TEST_DATA_DIR /
'configs/CONFIG_MAGIC_CORRUPT'))
test_file.processed_analysis['file_type'] = dict(mime='text/plain')
assert not self.analysis_plugin.probably_kernel_config(
test_file.binary)
def _unpack_fallback_check(self, fallback_mime, fallback_plugin_name):
broken_zip = FileObject(file_path=os.path.join(get_test_data_dir(), 'container/broken.zip'))
self.unpacker.unpack(broken_zip)
self.assertEqual(broken_zip.processed_analysis['unpacker']['0_ERROR_7z'][0:6], '\n7-Zip')
self.assertEqual(broken_zip.processed_analysis['unpacker']['0_FALLBACK_7z'], '7z (failed) -> {} (fallback)'.format(fallback_mime))
self.assertEqual(broken_zip.processed_analysis['unpacker']['plugin_used'], fallback_plugin_name)
return broken_zip
def generate_and_store_file_objects(self, file_paths: List[Path],
extractor_dir: str,
parent: FileObject):
extracted_files = {}
for item in file_paths:
if not file_is_empty(item):
current_file = FileObject(file_path=str(item))
current_virtual_path = '{}|{}|{}'.format(
parent.get_base_of_virtual_path(
parent.get_virtual_file_paths()[
parent.get_root_uid()][0]), parent.get_uid(),
get_object_path_excluding_fact_dirs(
make_unicode_string(str(item)),
str(Path(extractor_dir, 'files'))))
current_file.temporary_data[
'parent_fo_type'] = get_file_type_from_path(
parent.file_path)['mime']
if current_file.get_uid(
) in extracted_files: # the same file is extracted multiple times from one archive
extracted_files[current_file.get_uid()].virtual_file_path[
parent.get_root_uid()].append(current_virtual_path)
else:
self.db_interface.set_unpacking_lock(current_file.uid)
self.file_storage_system.store_file(current_file)
current_file.virtual_file_path = {
parent.get_root_uid(): [current_virtual_path]
}
current_file.parent_firmware_uids.add(
parent.get_root_uid())
extracted_files[current_file.get_uid()] = current_file
return extracted_files
def test_add_file_to_current_analyses(self):
self.scheduler.currently_running = {'parent_uid': ['foo', 'bar']}
fo = FileObject(binary=b'foo')
fo.parent_firmware_uids = {'parent_uid'}
fo.files_included = ['bar', 'new']
self.scheduler._add_to_current_analyses(fo)
assert sorted(self.scheduler.currently_running['parent_uid']) == ['bar', 'foo', 'new']
