def setUpClass(self):
test_init_dir = os.path.join(get_dir_of_file(__file__), 'data')
test_files = {
'systemd': 'etc/systemd/system/foobar',
'inittab': 'etc/inittab',
'rclocal': 'etc/rc.local',
'upstart': 'etc/init/baz.conf',
'runit': 'etc/service/lighttpd/run',
'runit_symlink': 'etc/service/example/run',
'runit_origin': 'etc/sv/example/run',
'only_comments': 'etc/inittab.invalid',
'initd': 'etc/init.d/skeleton',
'README': 'etc/init.d/README',
'initscript': 'etc/initscript'
}
for test_file, path in test_files.items():
exec("self.test_file_" + test_file +
" = FileObject(file_path=os.path.join(test_init_dir, path))")
exec("self.test_file_" + test_file +
".processed_analysis['file_type'] = {'mime': 'text/plain'}")
exec("self.test_file_" + test_file +
".root_uid = self.test_file_" + test_file + ".uid")
exec("self.test_file_" + test_file +
".virtual_file_path = {self.test_file_" + test_file +
".get_root_uid(): [\"" + path + "\"]}")
self.test_file_not_text = FileObject(
file_path="{}etc/systemd/system/foobar".format(test_init_dir))
self.test_file_not_text.processed_analysis['file_type'] = {
'mime': 'application/zip'
}
def setUpClass(cls):
test_init_dir = os.path.join(get_dir_of_file(__file__), 'data')
test_files = {
'systemd': 'etc/systemd/system/foobar',
'inittab': 'etc/inittab',
'rclocal': 'etc/rc.local',
'upstart': 'etc/init/baz.conf',
'runit': 'etc/service/lighttpd/run',
'runit_symlink': 'etc/service/example/run',
'runit_origin': 'etc/sv/example/run',
'only_comments': 'etc/inittab.invalid',
'initd': 'etc/init.d/skeleton',
'README': 'etc/init.d/README',
'initscript': 'etc/initscript'
}
for test_file, path in test_files.items():
test_fo = FileObject(file_path=os.path.join(test_init_dir, path))
setattr(cls, 'test_file_{}'.format(test_file), test_fo)
test_fo.processed_analysis['file_type'] = {'mime': 'text/plain'}
test_fo.root_uid = test_fo.uid
test_fo.virtual_file_path = {test_fo.get_root_uid(): [path]}
cls.test_file_not_text = FileObject(
file_path='{}etc/systemd/system/foobar'.format(test_init_dir))
cls.test_file_not_text.processed_analysis['file_type'] = {
'mime': 'application/zip'
}
def test_interactive_shell_command_none_correct_input(self):
script_path = os.path.join(get_dir_of_file(__file__),
'data/interactive.sh')
output, ret_code = execute_interactive_shell_command(script_path,
timeout=2)
assert 'give me some input' in output
assert '\n\nError: Execution timed out!' in output
assert ret_code > 0
def _get_view_file_path(self, plugin_path):
plugin_path = get_parent_dir(get_dir_of_file(plugin_path))
view_files = get_files_in_dir(os.path.join(plugin_path, 'view'))
if len(view_files) < 1:
logging.debug('{}: No view available! Generic view will be used.'.format(self.NAME))
return None
elif len(view_files) > 1:
logging.warning('{}: Plug-in provides more than one view! \'{}\' is used!'.format(self.NAME, view_files[0]))
return view_files[0]
def _set_rule_file_pathes(self, yara_uri_rules, yara_ip_rules):
internal_signature_dir = os.path.join(get_dir_of_file(__file__), 'yara_rules')
if yara_ip_rules is None:
self.yara_ip_rules = os.path.join(internal_signature_dir, 'ip_rules.yara')
else:
self.yara_ip_rules = yara_ip_rules
if yara_uri_rules is None:
self.yara_uri_rules = os.path.join(internal_signature_dir, 'uri_rules.yara')
else:
self.yara_uri_rules = yara_uri_rules
class CryptoCodeMaterialTest(AbstractSignatureTest):
PLUGIN_NAME = 'crypto_material'
TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data')
def setUp(self):
super().setUp()
config = self.init_basic_config()
self.analysis_plugin = AnalysisPlugin(self, config=config)
def test_gnupg(self):
self._rule_match('0x6C2DF2C5-pub.asc', 'PgpPublicKeyBlock',
len(['PgpPublicKeyBlock', 'PgpPublicKeyBlock_GnuPG']))
def test_ssh_public(self):
self._rule_match('id_rsa.pub', 'SshRsaPublicKeyBlock')
def test_ssh_private(self):
self._rule_match('id_rsa',
'SshRsaPrivateKeyBlock',
expected_number_of_rules=2)
def test_ssh_private_encrypted(self):
self._rule_match('id_rsa_encrypted',
'SshEncryptedRsaPrivateKeyBlock',
expected_number_of_rules=2)
def test_PKCS8(self):
self._rule_match('pkcs', 'Pkcs8PrivateKey', expected_number_of_rules=2)
def test_PKCS12(self):
self._rule_match('pkcs12', 'Pkcs12Certificate')
def test_SSL_key(self):
self._rule_match('ssl.key',
'SSLPrivateKey',
expected_number_of_rules=2)
def test_SSL_cert(self):
self._rule_match('ssl.crt', 'SSLCertificate')
def test_generic_public_key(self):
self._rule_match('generic_public_key', 'genericPublicKey')
def test_no_false_positives(self):
self._rule_match('FP_test', None, 0)
def test_der_error(self):
self._rule_match('error.der', None, 0)
def test_false_positive_ssl_cert(self):
self._rule_match('ssl_fp.file', None, 0)
def test_false_positive_pkcs_cert(self):
self._rule_match('pkcs_fp.file', None, 0)
def _init_plugins(self):
self.plugin_base = PluginBase(
package='filter_plugins.{}'.format(self.FILTER_TYPE))
self.filter_plugins = dict()
self.plugin_source = self.plugin_base.make_plugin_source(searchpath=[
os.path.join(get_dir_of_file(__file__),
'../filter_plugins/{}'.format(self.FILTER_TYPE))
])
plugin_list = self.plugin_source.list_plugins()
for item in plugin_list:
plugin = self.plugin_source.load_plugin(item)
plugin.setup(self)
def test_interactive_shell_command(self):
script_path = os.path.join(get_dir_of_file(__file__),
'data/interactive.sh')
expected_inputs = {
'give me some input:\r\n': 'test_input_1',
'give me more:\r\n': 'test_input_2'
}
output, ret_code = execute_interactive_shell_command(
script_path, inputs=expected_inputs, timeout=5)
assert 'first=test_input_1' in output
assert 'second=test_input_2' in output
assert ret_code == 0
import os
import re
from common_helper_files import get_dir_of_file
from analysis.YaraPluginBase import YaraBasePlugin
from helperFunctions.dataConversion import make_unicode_string
from helperFunctions.tag import TagColor
from plugins.analysis.software_components.bin import OS_LIST
SIGNATURE_DIR = os.path.join(get_dir_of_file(__file__), '../signatures')
class AnalysisPlugin(YaraBasePlugin):
'''
This plugin identifies software components
Credits:
OS Tagging functionality created by Roman Konertz during Firmware Bootcamp WT17/18 at University of Bonn
Maintained by Fraunhofer FKIE
'''
NAME = 'software_components'
DESCRIPTION = 'identify software components'
VERSION = '0.3'
FILE = __file__
def __init__(self, plugin_administrator, config=None, recursive=True):
super().__init__(plugin_administrator,
config=config,
recursive=recursive,
plugin_path=__file__)
import os
from common_helper_files import get_dir_of_file
from helperFunctions.yara_signature_testing import SignatureTestingMatching, SignatureTestingMeta
TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data')
SIGNATURE_PATH = os.path.join(get_dir_of_file(__file__), '../signatures/')
TEST_SIGNATURE_PATH = os.path.join(get_dir_of_file(__file__), '../test/data/')
class TestSoftwareSignatureMeta:
@classmethod
def setup_class(cls):
cls.sigTest = SignatureTestingMeta()
def test_check_meta_fields(self):
missing_fields = self.sigTest.check_meta_fields(SIGNATURE_PATH)
assert not missing_fields, 'Missing meta fields: {}'.format(missing_fields.__str__())
def test_check_meta_fields_missing(self):
missing_fields = self.sigTest.check_meta_fields(TEST_SIGNATURE_PATH)
assert len(missing_fields) == 3
assert all(
entry in missing_fields
for entry in ['website in missing_meta_1', 'description in missing_meta_1', 'ALL in missing_meta_2']
)
class TestAllKnownSoftwareMatched:
'''
import argparse
import logging
import sys
import os
from common_helper_files import get_dir_of_file
from main.blacklist_generator import create_blacklist
PROGRAM_NAME = 'Recovery Sort - Blacklist Generator'
PROGRAM_VERSION = '0.1'
PROGRAM_DESCRIPTION = 'Generate blacklists to be used with Recovery Sort'
default_out_file = os.path.join(get_dir_of_file(__file__), 'blacklist/user_generated_blacklist')
def _setup_argparser():
parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages')
parser.add_argument('input_dir', help='input directory')
parser.add_argument('-o', '--out_file', default=default_out_file, help='blacklist_file')
return parser.parse_args()
def _setup_logging(args):
log_format = logging.Formatter(fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger('')
if args.debug:
from unittest import TestCase
import docker
import pytest
from common_helper_files import get_dir_of_file
from requests.exceptions import ConnectionError as RequestConnectionError
from requests.exceptions import ReadTimeout
from test.common_helper import create_test_firmware, get_config_for_testing, get_test_data_dir
from test.mock import mock_patch
from test.unit.analysis.analysis_plugin_test_class import AnalysisPluginTest
from ..code import qemu_exec
from ..code.qemu_exec import EXECUTABLE
TEST_DATA_DIR = Path(get_dir_of_file(__file__)) / 'data/test_tmp_dir'
TEST_DATA_DIR_2 = Path(get_dir_of_file(__file__)) / 'data/test_tmp_dir_2'
TEST_DATA_DIR_3 = Path(get_dir_of_file(__file__)) / 'data/other_architectures'
CLI_PARAMETERS = ['-h', '--help', '-help', '--version', ' ']
class MockTmpDir:
def __init__(self, name):
self.name = name
def cleanup(self):
pass
class MockUnpacker:
tmp_dir = None
import os
import gc
import pytest
from common_helper_files import get_dir_of_file
from plugins.analysis.firmadyne.internal.helper import ResultType
from plugins.analysis.firmadyne.internal.firmadyne_wrapper import clean_firmadyne, execute_firmadyne
from plugins.analysis.firmadyne.internal.steps.prepare import extract_image
from plugins.analysis.firmadyne.internal.steps.analysis import start_analysis, match_unique_exploit_log_files, get_list_of_sorted_lines_from_text_file, transform_string_of_paths_into_jstree_structure, \
parse_logfile_list, start_nmap_analysis, start_metasploit_analysis, start_web_access_analysis, start_snmp_walk, execute_analysis_scripts
from plugins.analysis.firmadyne.internal.steps.emulation import network_is_available
TEST_FILE_PATH = os.path.join(get_dir_of_file(__file__), 'data')
def teardown_module(module):
gc.collect()
@pytest.mark.parametrize('input_data, expected', [
('', 'Failed'),
(os.path.join(TEST_FILE_PATH, 'WNAP320 Firmware Version 2.0.3.zip'), 'Successful')
])
def test_extract_image(input_data, expected):
clean_firmadyne()
result = extract_image(input_data)
assert result['extraction'] == expected
clean_firmadyne()
def load_word_list():
word_list_file = os.path.join(
get_parent_dir(get_dir_of_file(__file__)),
'internal/optimized_word_list.txt')
return get_string_list_from_file(word_list_file)
import os
from contextlib import suppress
from pathlib import Path
from test.common_helper import create_test_firmware
from test.unit.analysis.analysis_plugin_test_class import AnalysisPluginTest
from unittest import TestCase
from zlib import decompress
import pytest
from common_helper_files import get_dir_of_file
from helperFunctions.config import get_config_for_testing
from helperFunctions.fileSystem import get_test_data_dir
from ..code import qemu_exec
TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data/test_tmp_dir')
TEST_DATA_DIR_2 = os.path.join(get_dir_of_file(__file__), 'data/test_tmp_dir_2')
class MockTmpDir:
def __init__(self, name):
self.name = name
def cleanup(self):
pass
class MockUnpacker:
tmp_dir = None
def unpack_fo(self, _):
import json
import logging
import os
import re
from difflib import SequenceMatcher
import lief
from common_helper_files import get_dir_of_file
from analysis.PluginBase import AnalysisBasePlugin
from helperFunctions.hash import normalize_lief_items
from helperFunctions.tag import TagColor
TEMPLATE_FILE_PATH = os.path.join(get_dir_of_file(__file__), '../internal/matching_template.json')
class AnalysisPlugin(AnalysisBasePlugin):
NAME = 'elf_analysis'
DESCRIPTION = 'Analyzes and tags ELF executables and libraries'
DEPENDENCIES = ['file_type']
VERSION = '0.3'
MIME_WHITELIST = ['application/x-executable', 'application/x-object', 'application/x-sharedlib']
def __init__(self, plugin_adminstrator, config=None, recursive=True, offline_testing=False):
self.config = config
super().__init__(plugin_adminstrator, config=config, recursive=recursive, plugin_path=__file__, offline_testing=offline_testing)
def process_object(self, file_object):
elf_dict, parsed_binary = self._analyze_elf(file_object)
file_object.processed_analysis[self.NAME] = {'Output': elf_dict}
import json
import logging
import os
import re
from difflib import SequenceMatcher
import lief
from common_helper_files import get_dir_of_file
from analysis.PluginBase import AnalysisBasePlugin
from helperFunctions.hash import normalize_lief_items
from helperFunctions.tag import TagColor
TEMPLATE_FILE_PATH = os.path.join(get_dir_of_file(__file__),
'../internal/matching_template.json')
class AnalysisPlugin(AnalysisBasePlugin):
NAME = 'elf_analysis'
DESCRIPTION = 'Analyzes and tags ELF executables and libraries'
DEPENDENCIES = ['file_type']
VERSION = '0.3'
MIME_WHITELIST = [
'application/x-executable', 'application/x-object',
'application/x-sharedlib'
]
def __init__(self,
plugin_adminstrator,
config=None,
def test_get_dir_of_file_absolute_path(self):
test_file_path = os.path.join(self.tmp_dir.name, 'test_file')
write_binary_to_file('test', test_file_path)
absolute_file_path_result = get_dir_of_file(test_file_path)
self.assertEqual(absolute_file_path_result, self.tmp_dir.name)
import os
import pytest
import time
from common_helper_files import get_dir_of_file
from helper.meta import generate_uid, get_file_size, get_file_mime,\
get_file_name, get_modification_date
TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), '../data')
TEST_FILE_PATH = os.path.join(TEST_DATA_DIR, 'small_image.png')
@pytest.mark.parametrize('input_data, expected', [
(TEST_FILE_PATH, 'ed1015323e7c3a16936523ce1a64928a805f5b37534d74b596570d6931dc5684_159'),
('none_existing_file', '0_0')
])
def test_generate_uid(input_data, expected):
assert generate_uid(input_data) == expected
@pytest.mark.parametrize('input_data, expected', [(TEST_FILE_PATH, 159), ('none_existing_file', 0)])
def test_get_file_size(input_data, expected):
assert get_file_size(input_data) == expected
@pytest.mark.parametrize('input_data, expected', [(TEST_FILE_PATH, 'image/png'), ('none_existing_file', 'unknown')])
def test_get_file_mime(input_data, expected):
assert get_file_mime(input_data) == expected
@pytest.mark.parametrize('input_data, expected', [('foo/bar.img', 'bar.img'), ('/bar.img', 'bar.img'), ('bar.img', 'bar.img')])
import os
from tempfile import NamedTemporaryFile
from common_helper_files import get_dir_of_file
from ..internal.extract_os_names import extract_names, get_software_names
TEST_SIGNATURE_FILE = os.path.join(get_dir_of_file(__file__),
'./data/signatures/test_signature.yara')
def test_get_scanned_software():
assert get_software_names(TEST_SIGNATURE_FILE) == ['OS1', 'OS2']
def test_extract_names():
target_file = NamedTemporaryFile()
extract_names(TEST_SIGNATURE_FILE, target_file.name)
with open(target_file.name, 'r') as fd:
data = fd.read()
assert data == 'OS_LIST = ["OS1", "OS2"]\n'
from common_helper_files import get_dir_of_file
from common_helper_process import execute_shell_command
import json
import os
import sys
import logging
from analysis.PluginBase import AnalysisBasePlugin
INTERNAL_DIRECTORY_PATH = os.path.join(get_dir_of_file(__file__),
'../internal')
FIRMADYNE_INSTALLATION_DIR = os.path.join(get_dir_of_file(__file__),
'../bin/firmadyne')
class AnalysisPlugin(AnalysisBasePlugin):
NAME = 'firmadyne'
DEPENDENCIES = ['file_type']
DESCRIPTION = 'Dynamic Firmware Analysis utilizing Firmadyne'
VERSION = '0.4'
def __init__(self,
plugin_administrator,
config=None,
timeout=600,
recursive=True):
super().__init__(plugin_administrator,
config=config,
timeout=timeout,
no_multithread=True,
import logging
import os
import geoip2.database
from geoip2.errors import AddressNotFoundError
from maxminddb.errors import InvalidDatabaseError
from common_helper_files import get_dir_of_file
from analysis.PluginBase import AnalysisBasePlugin
from common_analysis_ip_and_uri_finder import CommonAnalysisIPAndURIFinder, ip_and_uri_finder_analysis
geoip_database_path = os.path.join(get_dir_of_file(__file__),
'../bin/GeoLite2-City/GeoLite2-City.mmdb')
class AnalysisPlugin(AnalysisBasePlugin):
'''
This plug-in finds IPs and URIs
'''
NAME = 'ip_and_uri_finder'
DEPENDENCIES = []
DESCRIPTION = 'search for IPs and URIs'
VERSION = ip_and_uri_finder_analysis.system_version
def __init__(self, plugin_administrator, config=None, recursive=True):
self.config = config
# additional init stuff can go here
self.IPAndURIFinder = CommonAnalysisIPAndURIFinder()
#! /usr/bin/env python3
import argparse
from collections import OrderedDict
from common_helper_process import execute_shell_command, execute_interactive_shell_command
import json
import os
import sys
import logging
from common_helper_files import get_dir_of_file
INTERNAL_DIRECTORY_PATH = os.path.join(get_dir_of_file(__file__))
sys.path.append(INTERNAL_DIRECTORY_PATH)
from helper import FIRMADYNE_PATH, ResultType, change_dir_to_firmadyne_dir
from steps.prepare import prepare_emulation
from steps.emulation import start_emulation
from steps.analysis import start_analysis
PROGRAM_NAME = 'Firmadyne Wrapper'
PROGRAM_VERSION = '0.4'
PROGRAM_DESCRIPTION = 'Automates firmadyne execution and stores result as json file'
def run_firmadyne_and_store_result(input_file, result_file_path):
execution_result, result_dict = execute_firmadyne(input_file)
if execution_result == ResultType.SUCCESS:
result_dict['result'] = 'Firmadyne finished all Steps succesfully!'
else:
result_dict['result'] = 'Firmadyne failed!'
import os
import sys
from common_helper_files import get_dir_of_file
from analysis.PluginBase import AnalysisBasePlugin
THIS_FILE_DIR = get_dir_of_file(__file__)
sys.path.append(os.path.join(THIS_FILE_DIR, '..', 'internal'))
from string_eval import eval_strings
class AnalysisPlugin(AnalysisBasePlugin):
'''
Sort strings by relevance
Credits:
Original version by Paul Schiffer created during Firmware Bootcamp WT16/17 at University of Bonn
Refactored and improved by Fraunhofer FKIE
'''
NAME = 'string_evaluator'
DEPENDENCIES = ['printable_strings']
DESCRIPTION = 'Tries to sort strings based on usefulness'
VERSION = '0.2'
def __init__(self,
plugin_administrator,
config=None,
recursive=True,
timeout=300):
super().__init__(plugin_administrator,
def _get_blacklist_dir():
return os.path.join(get_dir_of_file(__file__), '../../blacklist')
import unittest
from common_helper_files import get_dir_of_file
import os
from helperFunctions.yara_signature_testing import SignatureTestingMatching, SignatureTestingMeta
TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data')
SIGNATURE_PATH = os.path.join(get_dir_of_file(__file__), "../signatures/")
class TestSoftwareSignatureMeta(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.sigTest = SignatureTestingMeta()
def test_check_meta_fields(self):
missing_fields = self.sigTest.check_meta_fields(SIGNATURE_PATH)
self.assertEqual(
len(missing_fields), 0,
"Missing meta fields: {}".format(missing_fields.__str__()))
class TestAllKnownSoftwareMatched(unittest.TestCase):
def setUp(self):
self.sig_tester = SignatureTestingMatching()
def test_all_signatures_matched(self):
diff = self.sig_tester.check(
SIGNATURE_PATH,
os.path.join(TEST_DATA_DIR, "software_component_test_list.txt"))
self.assertEqual(diff, set(), "missing Signature for {}".format(diff))
def test_get_dir_of_file_relative_path(self):
relative_path_result = get_dir_of_file("test/some_file")
expected_result = os.path.join(os.getcwd(), "test")
self.assertEqual(relative_path_result, expected_result)
