def setUpClass(self): test_init_dir = os.path.join(get_dir_of_file(__file__), 'data') test_files = { 'systemd': 'etc/systemd/system/foobar', 'inittab': 'etc/inittab', 'rclocal': 'etc/rc.local', 'upstart': 'etc/init/baz.conf', 'runit': 'etc/service/lighttpd/run', 'runit_symlink': 'etc/service/example/run', 'runit_origin': 'etc/sv/example/run', 'only_comments': 'etc/inittab.invalid', 'initd': 'etc/init.d/skeleton', 'README': 'etc/init.d/README', 'initscript': 'etc/initscript' } for test_file, path in test_files.items(): exec("self.test_file_" + test_file + " = FileObject(file_path=os.path.join(test_init_dir, path))") exec("self.test_file_" + test_file + ".processed_analysis['file_type'] = {'mime': 'text/plain'}") exec("self.test_file_" + test_file + ".root_uid = self.test_file_" + test_file + ".uid") exec("self.test_file_" + test_file + ".virtual_file_path = {self.test_file_" + test_file + ".get_root_uid(): [\"" + path + "\"]}") self.test_file_not_text = FileObject( file_path="{}etc/systemd/system/foobar".format(test_init_dir)) self.test_file_not_text.processed_analysis['file_type'] = { 'mime': 'application/zip' }
def setUpClass(cls): test_init_dir = os.path.join(get_dir_of_file(__file__), 'data') test_files = { 'systemd': 'etc/systemd/system/foobar', 'inittab': 'etc/inittab', 'rclocal': 'etc/rc.local', 'upstart': 'etc/init/baz.conf', 'runit': 'etc/service/lighttpd/run', 'runit_symlink': 'etc/service/example/run', 'runit_origin': 'etc/sv/example/run', 'only_comments': 'etc/inittab.invalid', 'initd': 'etc/init.d/skeleton', 'README': 'etc/init.d/README', 'initscript': 'etc/initscript' } for test_file, path in test_files.items(): test_fo = FileObject(file_path=os.path.join(test_init_dir, path)) setattr(cls, 'test_file_{}'.format(test_file), test_fo) test_fo.processed_analysis['file_type'] = {'mime': 'text/plain'} test_fo.root_uid = test_fo.uid test_fo.virtual_file_path = {test_fo.get_root_uid(): [path]} cls.test_file_not_text = FileObject( file_path='{}etc/systemd/system/foobar'.format(test_init_dir)) cls.test_file_not_text.processed_analysis['file_type'] = { 'mime': 'application/zip' }
def test_interactive_shell_command_none_correct_input(self): script_path = os.path.join(get_dir_of_file(__file__), 'data/interactive.sh') output, ret_code = execute_interactive_shell_command(script_path, timeout=2) assert 'give me some input' in output assert '\n\nError: Execution timed out!' in output assert ret_code > 0
def _get_view_file_path(self, plugin_path): plugin_path = get_parent_dir(get_dir_of_file(plugin_path)) view_files = get_files_in_dir(os.path.join(plugin_path, 'view')) if len(view_files) < 1: logging.debug('{}: No view available! Generic view will be used.'.format(self.NAME)) return None elif len(view_files) > 1: logging.warning('{}: Plug-in provides more than one view! \'{}\' is used!'.format(self.NAME, view_files[0])) return view_files[0]
def _set_rule_file_pathes(self, yara_uri_rules, yara_ip_rules): internal_signature_dir = os.path.join(get_dir_of_file(__file__), 'yara_rules') if yara_ip_rules is None: self.yara_ip_rules = os.path.join(internal_signature_dir, 'ip_rules.yara') else: self.yara_ip_rules = yara_ip_rules if yara_uri_rules is None: self.yara_uri_rules = os.path.join(internal_signature_dir, 'uri_rules.yara') else: self.yara_uri_rules = yara_uri_rules
class CryptoCodeMaterialTest(AbstractSignatureTest): PLUGIN_NAME = 'crypto_material' TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data') def setUp(self): super().setUp() config = self.init_basic_config() self.analysis_plugin = AnalysisPlugin(self, config=config) def test_gnupg(self): self._rule_match('0x6C2DF2C5-pub.asc', 'PgpPublicKeyBlock', len(['PgpPublicKeyBlock', 'PgpPublicKeyBlock_GnuPG'])) def test_ssh_public(self): self._rule_match('id_rsa.pub', 'SshRsaPublicKeyBlock') def test_ssh_private(self): self._rule_match('id_rsa', 'SshRsaPrivateKeyBlock', expected_number_of_rules=2) def test_ssh_private_encrypted(self): self._rule_match('id_rsa_encrypted', 'SshEncryptedRsaPrivateKeyBlock', expected_number_of_rules=2) def test_PKCS8(self): self._rule_match('pkcs', 'Pkcs8PrivateKey', expected_number_of_rules=2) def test_PKCS12(self): self._rule_match('pkcs12', 'Pkcs12Certificate') def test_SSL_key(self): self._rule_match('ssl.key', 'SSLPrivateKey', expected_number_of_rules=2) def test_SSL_cert(self): self._rule_match('ssl.crt', 'SSLCertificate') def test_generic_public_key(self): self._rule_match('generic_public_key', 'genericPublicKey') def test_no_false_positives(self): self._rule_match('FP_test', None, 0) def test_der_error(self): self._rule_match('error.der', None, 0) def test_false_positive_ssl_cert(self): self._rule_match('ssl_fp.file', None, 0) def test_false_positive_pkcs_cert(self): self._rule_match('pkcs_fp.file', None, 0)
def _init_plugins(self): self.plugin_base = PluginBase( package='filter_plugins.{}'.format(self.FILTER_TYPE)) self.filter_plugins = dict() self.plugin_source = self.plugin_base.make_plugin_source(searchpath=[ os.path.join(get_dir_of_file(__file__), '../filter_plugins/{}'.format(self.FILTER_TYPE)) ]) plugin_list = self.plugin_source.list_plugins() for item in plugin_list: plugin = self.plugin_source.load_plugin(item) plugin.setup(self)
def test_interactive_shell_command(self): script_path = os.path.join(get_dir_of_file(__file__), 'data/interactive.sh') expected_inputs = { 'give me some input:\r\n': 'test_input_1', 'give me more:\r\n': 'test_input_2' } output, ret_code = execute_interactive_shell_command( script_path, inputs=expected_inputs, timeout=5) assert 'first=test_input_1' in output assert 'second=test_input_2' in output assert ret_code == 0
import os import re from common_helper_files import get_dir_of_file from analysis.YaraPluginBase import YaraBasePlugin from helperFunctions.dataConversion import make_unicode_string from helperFunctions.tag import TagColor from plugins.analysis.software_components.bin import OS_LIST SIGNATURE_DIR = os.path.join(get_dir_of_file(__file__), '../signatures') class AnalysisPlugin(YaraBasePlugin): ''' This plugin identifies software components Credits: OS Tagging functionality created by Roman Konertz during Firmware Bootcamp WT17/18 at University of Bonn Maintained by Fraunhofer FKIE ''' NAME = 'software_components' DESCRIPTION = 'identify software components' VERSION = '0.3' FILE = __file__ def __init__(self, plugin_administrator, config=None, recursive=True): super().__init__(plugin_administrator, config=config, recursive=recursive, plugin_path=__file__)
import os from common_helper_files import get_dir_of_file from helperFunctions.yara_signature_testing import SignatureTestingMatching, SignatureTestingMeta TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data') SIGNATURE_PATH = os.path.join(get_dir_of_file(__file__), '../signatures/') TEST_SIGNATURE_PATH = os.path.join(get_dir_of_file(__file__), '../test/data/') class TestSoftwareSignatureMeta: @classmethod def setup_class(cls): cls.sigTest = SignatureTestingMeta() def test_check_meta_fields(self): missing_fields = self.sigTest.check_meta_fields(SIGNATURE_PATH) assert not missing_fields, 'Missing meta fields: {}'.format(missing_fields.__str__()) def test_check_meta_fields_missing(self): missing_fields = self.sigTest.check_meta_fields(TEST_SIGNATURE_PATH) assert len(missing_fields) == 3 assert all( entry in missing_fields for entry in ['website in missing_meta_1', 'description in missing_meta_1', 'ALL in missing_meta_2'] ) class TestAllKnownSoftwareMatched:
''' import argparse import logging import sys import os from common_helper_files import get_dir_of_file from main.blacklist_generator import create_blacklist PROGRAM_NAME = 'Recovery Sort - Blacklist Generator' PROGRAM_VERSION = '0.1' PROGRAM_DESCRIPTION = 'Generate blacklists to be used with Recovery Sort' default_out_file = os.path.join(get_dir_of_file(__file__), 'blacklist/user_generated_blacklist') def _setup_argparser(): parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION)) parser.add_argument('-V', '--version', action='version', version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION)) parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages') parser.add_argument('input_dir', help='input directory') parser.add_argument('-o', '--out_file', default=default_out_file, help='blacklist_file') return parser.parse_args() def _setup_logging(args): log_format = logging.Formatter(fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') logger = logging.getLogger('') if args.debug:
from unittest import TestCase import docker import pytest from common_helper_files import get_dir_of_file from requests.exceptions import ConnectionError as RequestConnectionError from requests.exceptions import ReadTimeout from test.common_helper import create_test_firmware, get_config_for_testing, get_test_data_dir from test.mock import mock_patch from test.unit.analysis.analysis_plugin_test_class import AnalysisPluginTest from ..code import qemu_exec from ..code.qemu_exec import EXECUTABLE TEST_DATA_DIR = Path(get_dir_of_file(__file__)) / 'data/test_tmp_dir' TEST_DATA_DIR_2 = Path(get_dir_of_file(__file__)) / 'data/test_tmp_dir_2' TEST_DATA_DIR_3 = Path(get_dir_of_file(__file__)) / 'data/other_architectures' CLI_PARAMETERS = ['-h', '--help', '-help', '--version', ' '] class MockTmpDir: def __init__(self, name): self.name = name def cleanup(self): pass class MockUnpacker: tmp_dir = None
import os import gc import pytest from common_helper_files import get_dir_of_file from plugins.analysis.firmadyne.internal.helper import ResultType from plugins.analysis.firmadyne.internal.firmadyne_wrapper import clean_firmadyne, execute_firmadyne from plugins.analysis.firmadyne.internal.steps.prepare import extract_image from plugins.analysis.firmadyne.internal.steps.analysis import start_analysis, match_unique_exploit_log_files, get_list_of_sorted_lines_from_text_file, transform_string_of_paths_into_jstree_structure, \ parse_logfile_list, start_nmap_analysis, start_metasploit_analysis, start_web_access_analysis, start_snmp_walk, execute_analysis_scripts from plugins.analysis.firmadyne.internal.steps.emulation import network_is_available TEST_FILE_PATH = os.path.join(get_dir_of_file(__file__), 'data') def teardown_module(module): gc.collect() @pytest.mark.parametrize('input_data, expected', [ ('', 'Failed'), (os.path.join(TEST_FILE_PATH, 'WNAP320 Firmware Version 2.0.3.zip'), 'Successful') ]) def test_extract_image(input_data, expected): clean_firmadyne() result = extract_image(input_data) assert result['extraction'] == expected clean_firmadyne()
def load_word_list(): word_list_file = os.path.join( get_parent_dir(get_dir_of_file(__file__)), 'internal/optimized_word_list.txt') return get_string_list_from_file(word_list_file)
import os from contextlib import suppress from pathlib import Path from test.common_helper import create_test_firmware from test.unit.analysis.analysis_plugin_test_class import AnalysisPluginTest from unittest import TestCase from zlib import decompress import pytest from common_helper_files import get_dir_of_file from helperFunctions.config import get_config_for_testing from helperFunctions.fileSystem import get_test_data_dir from ..code import qemu_exec TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data/test_tmp_dir') TEST_DATA_DIR_2 = os.path.join(get_dir_of_file(__file__), 'data/test_tmp_dir_2') class MockTmpDir: def __init__(self, name): self.name = name def cleanup(self): pass class MockUnpacker: tmp_dir = None def unpack_fo(self, _):
import json import logging import os import re from difflib import SequenceMatcher import lief from common_helper_files import get_dir_of_file from analysis.PluginBase import AnalysisBasePlugin from helperFunctions.hash import normalize_lief_items from helperFunctions.tag import TagColor TEMPLATE_FILE_PATH = os.path.join(get_dir_of_file(__file__), '../internal/matching_template.json') class AnalysisPlugin(AnalysisBasePlugin): NAME = 'elf_analysis' DESCRIPTION = 'Analyzes and tags ELF executables and libraries' DEPENDENCIES = ['file_type'] VERSION = '0.3' MIME_WHITELIST = ['application/x-executable', 'application/x-object', 'application/x-sharedlib'] def __init__(self, plugin_adminstrator, config=None, recursive=True, offline_testing=False): self.config = config super().__init__(plugin_adminstrator, config=config, recursive=recursive, plugin_path=__file__, offline_testing=offline_testing) def process_object(self, file_object): elf_dict, parsed_binary = self._analyze_elf(file_object) file_object.processed_analysis[self.NAME] = {'Output': elf_dict}
import json import logging import os import re from difflib import SequenceMatcher import lief from common_helper_files import get_dir_of_file from analysis.PluginBase import AnalysisBasePlugin from helperFunctions.hash import normalize_lief_items from helperFunctions.tag import TagColor TEMPLATE_FILE_PATH = os.path.join(get_dir_of_file(__file__), '../internal/matching_template.json') class AnalysisPlugin(AnalysisBasePlugin): NAME = 'elf_analysis' DESCRIPTION = 'Analyzes and tags ELF executables and libraries' DEPENDENCIES = ['file_type'] VERSION = '0.3' MIME_WHITELIST = [ 'application/x-executable', 'application/x-object', 'application/x-sharedlib' ] def __init__(self, plugin_adminstrator, config=None,
def test_get_dir_of_file_absolute_path(self): test_file_path = os.path.join(self.tmp_dir.name, 'test_file') write_binary_to_file('test', test_file_path) absolute_file_path_result = get_dir_of_file(test_file_path) self.assertEqual(absolute_file_path_result, self.tmp_dir.name)
import os import pytest import time from common_helper_files import get_dir_of_file from helper.meta import generate_uid, get_file_size, get_file_mime,\ get_file_name, get_modification_date TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), '../data') TEST_FILE_PATH = os.path.join(TEST_DATA_DIR, 'small_image.png') @pytest.mark.parametrize('input_data, expected', [ (TEST_FILE_PATH, 'ed1015323e7c3a16936523ce1a64928a805f5b37534d74b596570d6931dc5684_159'), ('none_existing_file', '0_0') ]) def test_generate_uid(input_data, expected): assert generate_uid(input_data) == expected @pytest.mark.parametrize('input_data, expected', [(TEST_FILE_PATH, 159), ('none_existing_file', 0)]) def test_get_file_size(input_data, expected): assert get_file_size(input_data) == expected @pytest.mark.parametrize('input_data, expected', [(TEST_FILE_PATH, 'image/png'), ('none_existing_file', 'unknown')]) def test_get_file_mime(input_data, expected): assert get_file_mime(input_data) == expected @pytest.mark.parametrize('input_data, expected', [('foo/bar.img', 'bar.img'), ('/bar.img', 'bar.img'), ('bar.img', 'bar.img')])
import os from tempfile import NamedTemporaryFile from common_helper_files import get_dir_of_file from ..internal.extract_os_names import extract_names, get_software_names TEST_SIGNATURE_FILE = os.path.join(get_dir_of_file(__file__), './data/signatures/test_signature.yara') def test_get_scanned_software(): assert get_software_names(TEST_SIGNATURE_FILE) == ['OS1', 'OS2'] def test_extract_names(): target_file = NamedTemporaryFile() extract_names(TEST_SIGNATURE_FILE, target_file.name) with open(target_file.name, 'r') as fd: data = fd.read() assert data == 'OS_LIST = ["OS1", "OS2"]\n'
from common_helper_files import get_dir_of_file from common_helper_process import execute_shell_command import json import os import sys import logging from analysis.PluginBase import AnalysisBasePlugin INTERNAL_DIRECTORY_PATH = os.path.join(get_dir_of_file(__file__), '../internal') FIRMADYNE_INSTALLATION_DIR = os.path.join(get_dir_of_file(__file__), '../bin/firmadyne') class AnalysisPlugin(AnalysisBasePlugin): NAME = 'firmadyne' DEPENDENCIES = ['file_type'] DESCRIPTION = 'Dynamic Firmware Analysis utilizing Firmadyne' VERSION = '0.4' def __init__(self, plugin_administrator, config=None, timeout=600, recursive=True): super().__init__(plugin_administrator, config=config, timeout=timeout, no_multithread=True,
import logging import os import geoip2.database from geoip2.errors import AddressNotFoundError from maxminddb.errors import InvalidDatabaseError from common_helper_files import get_dir_of_file from analysis.PluginBase import AnalysisBasePlugin from common_analysis_ip_and_uri_finder import CommonAnalysisIPAndURIFinder, ip_and_uri_finder_analysis geoip_database_path = os.path.join(get_dir_of_file(__file__), '../bin/GeoLite2-City/GeoLite2-City.mmdb') class AnalysisPlugin(AnalysisBasePlugin): ''' This plug-in finds IPs and URIs ''' NAME = 'ip_and_uri_finder' DEPENDENCIES = [] DESCRIPTION = 'search for IPs and URIs' VERSION = ip_and_uri_finder_analysis.system_version def __init__(self, plugin_administrator, config=None, recursive=True): self.config = config # additional init stuff can go here self.IPAndURIFinder = CommonAnalysisIPAndURIFinder()
#! /usr/bin/env python3 import argparse from collections import OrderedDict from common_helper_process import execute_shell_command, execute_interactive_shell_command import json import os import sys import logging from common_helper_files import get_dir_of_file INTERNAL_DIRECTORY_PATH = os.path.join(get_dir_of_file(__file__)) sys.path.append(INTERNAL_DIRECTORY_PATH) from helper import FIRMADYNE_PATH, ResultType, change_dir_to_firmadyne_dir from steps.prepare import prepare_emulation from steps.emulation import start_emulation from steps.analysis import start_analysis PROGRAM_NAME = 'Firmadyne Wrapper' PROGRAM_VERSION = '0.4' PROGRAM_DESCRIPTION = 'Automates firmadyne execution and stores result as json file' def run_firmadyne_and_store_result(input_file, result_file_path): execution_result, result_dict = execute_firmadyne(input_file) if execution_result == ResultType.SUCCESS: result_dict['result'] = 'Firmadyne finished all Steps succesfully!' else: result_dict['result'] = 'Firmadyne failed!'
import os import sys from common_helper_files import get_dir_of_file from analysis.PluginBase import AnalysisBasePlugin THIS_FILE_DIR = get_dir_of_file(__file__) sys.path.append(os.path.join(THIS_FILE_DIR, '..', 'internal')) from string_eval import eval_strings class AnalysisPlugin(AnalysisBasePlugin): ''' Sort strings by relevance Credits: Original version by Paul Schiffer created during Firmware Bootcamp WT16/17 at University of Bonn Refactored and improved by Fraunhofer FKIE ''' NAME = 'string_evaluator' DEPENDENCIES = ['printable_strings'] DESCRIPTION = 'Tries to sort strings based on usefulness' VERSION = '0.2' def __init__(self, plugin_administrator, config=None, recursive=True, timeout=300): super().__init__(plugin_administrator,
def _get_blacklist_dir(): return os.path.join(get_dir_of_file(__file__), '../../blacklist')
import unittest from common_helper_files import get_dir_of_file import os from helperFunctions.yara_signature_testing import SignatureTestingMatching, SignatureTestingMeta TEST_DATA_DIR = os.path.join(get_dir_of_file(__file__), 'data') SIGNATURE_PATH = os.path.join(get_dir_of_file(__file__), "../signatures/") class TestSoftwareSignatureMeta(unittest.TestCase): @classmethod def setUpClass(cls): cls.sigTest = SignatureTestingMeta() def test_check_meta_fields(self): missing_fields = self.sigTest.check_meta_fields(SIGNATURE_PATH) self.assertEqual( len(missing_fields), 0, "Missing meta fields: {}".format(missing_fields.__str__())) class TestAllKnownSoftwareMatched(unittest.TestCase): def setUp(self): self.sig_tester = SignatureTestingMatching() def test_all_signatures_matched(self): diff = self.sig_tester.check( SIGNATURE_PATH, os.path.join(TEST_DATA_DIR, "software_component_test_list.txt")) self.assertEqual(diff, set(), "missing Signature for {}".format(diff))
def test_get_dir_of_file_relative_path(self): relative_path_result = get_dir_of_file("test/some_file") expected_result = os.path.join(os.getcwd(), "test") self.assertEqual(relative_path_result, expected_result)