def test_get_src_dir_cwd(restore_cwd, working_directory): real_src_dir = get_src_dir() os.chdir(working_directory) assert os.path.exists('{}/helperFunctions/fileSystem.py'.format( real_src_dir)), 'fileSystem.py found in correct place' assert get_src_dir( ) == real_src_dir, 'same source dir before and after chdir'
def check_correct_src_dir(self, working_directory): real_src_dir = get_src_dir() os.chdir(working_directory) self.assertTrue( os.path.exists( '{}/helperFunctions/fileSystem.py'.format(real_src_dir)), 'fileSystem.py found in correct place') self.assertEqual(get_src_dir(), real_src_dir, 'same source dir before and after chdir')
def test_get_view_file_path(self): plugin_path = os.path.join(get_src_dir(), 'plugins/analysis/file_type/') code_path = os.path.join(plugin_path, 'code/file_type.py') estimated_view_path = os.path.join(plugin_path, 'view/file_type.html') assert self.pBase._get_view_file_path(code_path) == estimated_view_path plugin_path_without_view = os.path.join( get_src_dir(), 'plugins/analysis/dummy/code/dummy.py') assert self.pBase._get_view_file_path(plugin_path_without_view) is None
def test_start_script_help_and_version(script, expected_str): output, return_code = execute_shell_command_get_return_code('{} -h'.format( os.path.join(get_src_dir(), script)), timeout=5) assert return_code == 0 assert 'usage: {}'.format(script) in output output, return_code = execute_shell_command_get_return_code('{} -V'.format( os.path.join(get_src_dir(), script)), timeout=5) assert expected_str in output, 'Wrong output {}'.format(output) assert return_code == 0 gc.collect()
def test_start_script_help_and_version(script): output, return_code = execute_shell_command_get_return_code('{} -h'.format( os.path.join(get_src_dir(), script)), timeout=5) assert return_code == 0 assert 'usage: {}'.format(script) in output output, return_code = execute_shell_command_get_return_code('{} -V'.format( os.path.join(get_src_dir(), script)), timeout=5) assert output[0:5] == 'FACT ' assert return_code == 0 gc.collect()
def start_uwsgi_server(config_path=None): config_parameter = ' --pyargv {}'.format( config_path) if config_path else '' command = 'uwsgi --ini {}/uwsgi_config.ini{}'.format( get_config_dir(), config_parameter) process = Popen(split(command), cwd=get_src_dir()) return process
def _get_signature_file(self, plugin_path): if plugin_path: sig_file_name = self._get_signature_file_name(plugin_path) sig_dir = os.path.join(get_src_dir(), 'analysis/signatures') self.signature_path = os.path.join(sig_dir, sig_file_name) else: self.signature_path = None
def _create_variety_data(config): varietyjs_script_path = Path( get_src_dir()) / config['data_storage']['variety_path'] mongo_call = ( 'mongo --port {mongo_port} -u "{username}" -p "{password}" --authenticationDatabase "admin" ' .format( mongo_port=config['data_storage']['mongo_port'], username=config['data_storage']['db_admin_user'], password=config['data_storage']['db_admin_pw'], )) output, return_code = execute_shell_command_get_return_code( '{mongo_call} {database} --eval "var collection = \'file_objects\', persistResults=true" {script_path}' .format(mongo_call=mongo_call, database=config['data_storage']['main_database'], script_path=varietyjs_script_path), timeout=None) if return_code == 0: execute_shell_command( '{mongo_call} varietyResults --eval \'{command}\''.format( mongo_call=mongo_call, command= 'db.file_objectsKeys.deleteMany({"_id.key": {"$regex": "skipped|file_system_flag"}})' ), ) logging.debug(output) return return_code
def test_get_modules_in_path(self): plugin_dir_path = os.path.join(get_src_dir(), 'plugins') plugin_folder_modules = PluginRoutes._get_modules_in_path( plugin_dir_path) assert len(plugin_folder_modules) >= 3 for category in PLUGIN_CATEGORIES: assert category in plugin_folder_modules
def start_uwsgi_server(config_path=None): config_parameter = ' --pyargv {}'.format( config_path) if config_path else '' p = Popen('(cd {} && uwsgi --ini {}/uwsgi_config.ini{})'.format( get_src_dir(), get_config_dir(), config_parameter), shell=True) return p
def test_fact_complete_start(): output, return_code = execute_shell_command_get_return_code( '{} -d -t'.format(os.path.join(get_src_dir(), 'start_fact.py'))) assert '[DEBUG]' in output assert 'Analysis System online...' in output assert 'Analysis System offline' in output assert return_code == 0 gc.collect()
def setUp(self): super().setUp() config = self.init_basic_config() self.intended_signature_path = os.path.join(get_src_dir(), 'analysis/signatures', self.PLUGIN_NAME) self.analysis_plugin = YaraBasePlugin( self, config=config, plugin_path='/foo/bar/Yara_Base_Plugin/code/test.py')
def _get_plugin_src_dirs(base_dir): plug_in_base_path = Path(get_src_dir(), base_dir) plugin_dirs = get_dirs_in_dir(str(plug_in_base_path)) plugins = [] for plugin_path in plugin_dirs: plugin_code_dir = Path(plugin_path, 'code') if plugin_code_dir.is_dir(): plugins.append(str(plugin_code_dir)) else: logging.warning('Plugin has no code directory: {}'.format(plugin_path)) return plugins
def main(): _create_signature_dir() for plugin_dir in get_dirs_in_dir(os.path.join(get_src_dir(), 'plugins/analysis')): signature_dir = os.path.join(plugin_dir, 'signatures') if os.path.isdir(signature_dir): print('Compile signatures in {}'.format(signature_dir)) with NamedTemporaryFile(mode='w') as tmp_file: _create_joint_signature_file(signature_dir, tmp_file) _create_compiled_signature_file(signature_dir, tmp_file) return 0
def _create_variety_data(config): full_variety_path = os.path.join(get_src_dir(), config['data_storage']['variety_path']) output, return_code = execute_shell_command_get_return_code( 'mongo --port {mongo_port} {main_database} -u "{username}" -p "{password}" --authenticationDatabase "admin" --eval "var collection = \'file_objects\', persistResults=true" {script_path}' .format(mongo_port=config['data_storage']['mongo_port'], username=config['data_storage']['db_admin_user'], password=config['data_storage']['db_admin_pw'], main_database=config['data_storage']['main_database'], script_path=full_variety_path), timeout=None) logging.debug(output) return return_code
def _install_plugins(distribution, skip_docker, only_docker=False): installer_paths = Path(get_src_dir() + '/plugins/').glob('*/*/install.py') for install_script in installer_paths: plugin_name = install_script.parent.name plugin_type = install_script.parent.parent.name plugin = importlib.import_module(f'plugins.{plugin_type}.{plugin_name}.install') plugin_installer = plugin.Installer(distribution, skip_docker=skip_docker) logging.info(f'Installing {plugin_name} plugin.') if not only_docker: plugin_installer.install() else: plugin_installer.install_docker_images() logging.info(f'Finished installing {plugin_name} plugin.\n')
def _get_plugin_src_dirs(base_dir: str) -> List[str]: ''' Returns a list of all plugin code directories. E.g. if base_dir contains the qemu_exec plugin it would return `base_dir`/qemu_exec/code. :param base_dir: The root directory of all plugins ''' plug_in_base_path = Path(get_src_dir(), base_dir) plugin_dirs = get_dirs_in_dir(str(plug_in_base_path)) plugins = [] for plugin_path in plugin_dirs: if plugin_path.endswith('__pycache__'): continue plugin_code_dir = Path(plugin_path, 'code') if plugin_code_dir.is_dir(): plugins.append(str(plugin_code_dir)) else: logging.warning( 'Plugin has no code directory: {}'.format(plugin_path)) return plugins
def unpack_function(file_path, tmp_dir): script_path = path.join(get_src_dir(), "bin", "amba_fwpak.py") if not path.exists(script_path): return { 'output': "Error: phantom_firmware_tools not installed! Re-Run the installation script!" } fallback_directory = getcwd() chdir(tmp_dir) output = execute_shell_command('fakeroot {} -x -vv -m {}'.format( script_path, file_path)) + "\n" _rename_files(file_path) _remove_ini_files() chdir(fallback_directory) meta_data = {'output': output} logging.debug(output) return meta_data
import json import logging import re from pathlib import Path from common_helper_process import execute_shell_command_get_return_code from analysis.PluginBase import AnalysisBasePlugin from helperFunctions.fileSystem import get_src_dir SHELL_SCRIPT = Path(get_src_dir()) / 'bin' / 'checksec' class AnalysisPlugin(AnalysisBasePlugin): NAME = 'exploit_mitigations' DESCRIPTION = 'analyses ELF binaries within a firmware for present exploit mitigation techniques' DEPENDENCIES = ['file_type'] MIME_WHITELIST = ['application/x-executable', 'application/x-object', 'application/x-sharedlib'] VERSION = '0.1.6' def __init__(self, plugin_administrator, config=None, recursive=True): self.config = config if not SHELL_SCRIPT.is_file(): raise RuntimeError(f'checksec not found at path {SHELL_SCRIPT}. Please re-run the backend installation.') super().__init__(plugin_administrator, config=config, recursive=recursive, plugin_path=__file__) def process_object(self, file_object): try: if re.search(r'.*elf.*', file_object.processed_analysis['file_type']['full'].lower()) is not None:
def _load_view(): path = os.path.join( get_src_dir(), 'plugins/analysis/{}/routes/ajax_view.html'.format( AnalysisPlugin.NAME)) with open(path, "r") as fp: return fp.read()
from base64 import b64decode from contextlib import suppress from pathlib import Path from tempfile import NamedTemporaryFile from typing import Callable, List from common_helper_process import execute_shell_command from analysis.PluginBase import AnalysisBasePlugin from helperFunctions.fileSystem import get_src_dir from helperFunctions.tag import TagColor from objects.file import FileObject from plugins.mime_blacklists import MIME_BLACKLIST_NON_EXECUTABLE JOHN_PATH = Path(__file__).parent.parent / 'bin' / 'john' WORDLIST_PATH = Path(get_src_dir()) / 'bin' / 'passwords.txt' USER_NAME_REGEX = br'[a-zA-Z][a-zA-Z0-9_-]{2,15}' UNIX_REGEXES = [ USER_NAME_REGEX + br':[^:]?:\d+:\d*:[^:]*:[^:]*:[^\n ]*', USER_NAME_REGEX + br':\$[1256][ay]?\$[a-zA-Z0-9\./+]+\$[a-zA-Z0-9\./+]{16,128}={0,2}', # MD5 / Blowfish / SHA USER_NAME_REGEX + br':[a-zA-Z0-9\./=]{13}:\d*:\d*:' # DES ] HTPASSWD_REGEXES = [ USER_NAME_REGEX + br':\$apr1\$[a-zA-Z0-9\./+=]+\$[a-zA-Z0-9\./+]{22}', # MD5 apr1 USER_NAME_REGEX + br':\{SHA\}[a-zA-Z0-9\./+]{27}=', # SHA-1 ] MOSQUITTO_REGEXES = [ br'[a-zA-Z][a-zA-Z0-9_-]{2,15}\:\$6\$[a-zA-Z0-9+/=]+\$[a-zA-Z0-9+/]{86}==' ]
def _get_signature_file(self, plugin_path): sig_file_name = self._get_signature_file_name(plugin_path) return str(Path(get_src_dir()) / 'analysis/signatures' / sig_file_name)
class AnalysisPlugin(AnalysisBasePlugin): ''' This plug-in tries to find and crack passwords ''' NAME = 'users_and_passwords' DEPENDENCIES = [] MIME_BLACKLIST = ['audio', 'filesystem', 'image', 'video'] DESCRIPTION = 'search for UNIX, httpd, and mosquitto password files, parse them and try to crack the passwords' VERSION = '0.4.5' wordlist_path = os.path.join(get_src_dir(), 'bin/passwords.txt') def __init__(self, plugin_administrator, config=None, recursive=True): self.config = config super().__init__(plugin_administrator, config=config, recursive=recursive, no_multithread=True, plugin_path=__file__) def process_object(self, file_object): if self.NAME not in file_object.processed_analysis: file_object.processed_analysis[self.NAME] = {} file_object.processed_analysis[self.NAME]['summary'] = [] self.find_unix_entries(file_object) self.find_mosquitto_entries(file_object) return file_object def find_unix_entries(self, file_object): for passwd_regex in [ b'[a-zA-Z][a-zA-Z0-9_-]{2,15}:[^:]?:\\d+:\\d*:[^:]*:[^:]*:[^\n ]*', b'[a-zA-Z][a-zA-Z0-9_-]{2,15}:\\$[^\\$]+\\$[^\\$]+\\$[a-zA-Z0-9\\./+]{16,128}={0,3}' ]: passwd_entries = re.findall(passwd_regex, file_object.binary) if passwd_entries: result = self._generate_analysis_entry(passwd_entries, file_object.uid) self.update_file_object(file_object, result) def find_mosquitto_entries(self, file_object): for passwd_regex in [br'[a-zA-Z][a-zA-Z0-9_-]{2,15}\:\$6\$[a-zA-Z0-9+/=]+\$[a-zA-Z0-9+/]{86}==']: passwd_entries = re.findall(passwd_regex, file_object.binary) if passwd_entries: result = self._generate_mosquitto_entry(passwd_entries) self.update_file_object(file_object, result) def _add_found_password_tag(self, file_object, result): for password_entry in result: if 'password' in result[password_entry]: self.add_analysis_tag( file_object, '{}_{}'.format(password_entry, result[password_entry]['password']), 'Password: {}:{}'.format(password_entry, result[password_entry]['password']), TagColor.RED, True ) def update_file_object(self, file_object, result_entry): file_object.processed_analysis[self.NAME].update(result_entry) file_object.processed_analysis[self.NAME]['summary'] += list(result_entry.keys()) self._add_found_password_tag(file_object, result_entry) def _generate_analysis_entry(self, passwd_entries, uid: str): result = {} for entry in [e.split(b':') for e in passwd_entries]: key = entry[0].decode(encoding='utf_8', errors='replace') result_entry = result['{}:unix'.format(key)] = {} result_entry['type'] = 'unix' result_entry['entry'] = b':'.join(entry).decode(encoding='utf_8', errors='replace') try: if entry[1][0] == ord('$'): result_entry['password-hash'] = entry[1].decode(encoding='utf_8', errors='replace') cracked_pw = self._crack_hash(b':'.join(entry[:2]), result_entry) result_entry['cracked'] = bool(cracked_pw) except (IndexError, AttributeError, TypeError): logging.warning('Unsupported Format: {}'.format(uid), exc_info=True) return result def _generate_mosquitto_entry(self, passwd_entries): result = {} for entry in [m.split(b'$') for m in passwd_entries]: user = entry[0].decode(encoding='utf_8', errors='replace')[:-1] salt_hash = entry[2].decode(encoding='utf_8', errors='replace') passwd_hash = entry[3].decode(encoding='utf_8', errors='replace') passwd_entry = '{}:$dynamic_82${}$HEX${}'.format(user, b64decode(passwd_hash).hex(), b64decode(salt_hash).hex()) result_entry = result['{}:mosquitto'.format(user)] = {} result_entry['type'] = 'mosquitto' result_entry['entry'] = b'$'.join(entry).decode(encoding='utf_8', errors='replace') result_entry['password-hash'] = passwd_hash cracked_pw = self._crack_hash(passwd_entry.encode(), result_entry, '--format=dynamic_82') result_entry['cracked'] = bool(cracked_pw) return result def _crack_hash(self, passwd_entry, result_entry, format_term=''): with NamedTemporaryFile() as fp: fp.write(passwd_entry) fp.seek(0) result_entry['log'] = execute_shell_command('{} --wordlist={} {} {}'.format(JOHN_PATH, self.wordlist_path, fp.name, format_term)) output = execute_shell_command('{} {} --show {}'.format(JOHN_PATH, fp.name, format_term)).split('\n') if len(output) > 1: with suppress(KeyError): if '0 password hashes cracked' in output[-2]: result_entry['ERROR'] = 'hash type is not supported' return False result_entry['password'] = output[0].split(':')[1] return True return False
class AnalysisPlugin(AnalysisBasePlugin): ''' This Plugin trys to find and crack passwords ''' NAME = 'users_and_passwords' DEPENDENCIES = [] MIME_BLACKLIST = ['audio', 'filesystem', 'image', 'video'] DESCRIPTION = 'search for UNIX and httpd password files, parse them and try to crack the passwords' VERSION = '0.4.1' wordlist_path = os.path.join(get_src_dir(), 'bin/passwords.txt') def __init__(self, plugin_administrator, config=None, recursive=True): ''' recursive flag: If True recursively analyze included files default flags should be edited above. Otherwise the scheduler cannot overwrite them. ''' self.config = config # additional init stuff can go here super().__init__(plugin_administrator, config=config, recursive=recursive, no_multithread=True, plugin_path=__file__) def process_object(self, file_object): ''' This function must be implemented by the plugin. Analysis result must be a dict stored in file_object.processed_analysis[self.NAME] If you want to propagate results to parent objects store a list of strings 'summary' entry of your result dict ''' if self.NAME not in file_object.processed_analysis: file_object.processed_analysis[self.NAME] = {} file_object.processed_analysis[self.NAME]['summary'] = [] for passwd_regex in [ b'[a-zA-Z][a-zA-Z0-9_-]{2,15}:[^:]?:\\d+:\\d*:[^:]*:[^:]*:[^\n ]*', b'[a-zA-Z][a-zA-Z0-9_-]{2,15}:\\$[^\\$]+\\$[^\\$]+\\$[a-zA-Z0-9\\./]{16,128}' ]: passwd_entries = re.findall(passwd_regex, file_object.binary) if passwd_entries: result = self._generate_analysis_entry(passwd_entries) file_object.processed_analysis[self.NAME].update(result) file_object.processed_analysis[self.NAME]['summary'] += list( result.keys()) return file_object def _generate_analysis_entry(self, passwd_entries): result = {} for entry in [e.split(b':') for e in passwd_entries]: key = entry[0].decode(encoding='utf_8', errors='replace') result[key] = { 'entry': b':'.join(entry).decode(encoding='utf_8', errors='replace') } try: if entry[1][0] == ord('$'): result[key]['password-hash'] = entry[1].decode( encoding='utf_8', errors='replace') cracked_pw = self._crack_hash(entry, result, key) result[key]['cracked'] = True if cracked_pw else False except Exception as e: logging.error('Invalid Format: {} - {}'.format( sys.exc_info()[0].__name__, e)) return result def _crack_hash(self, passwd_entry, result_dict, key): with NamedTemporaryFile() as fp: fp.write(b':'.join(passwd_entry[:2])) fp.seek(0) result_dict[key]['log'] = execute_shell_command( 'john --wordlist={} {}'.format(self.wordlist_path, fp.name)) output = execute_shell_command('john --show {}'.format( fp.name)).split('\n') if len(output) > 2: with suppress(KeyError): result_dict[key]['password'] = output[0].split(':')[1] return True return False
def get_test_data_dir(): ''' Returns the absolute path of the test data directory ''' return os.path.join(get_src_dir(), 'test/data')
def get_config_dir(): ''' Returns the absolute path of the config directory ''' return '{}/config'.format(get_src_dir())
import importlib import inspect import pkgutil from flask_restful import Resource from helperFunctions.fileSystem import get_src_dir from web_interface.components.component_base import ComponentBase ROUTES_MODULE_NAME = 'routes' PLUGIN_CATEGORIES = ['analysis', 'compare'] PLUGIN_DIR = '{}/plugins'.format(get_src_dir()) class PluginRoutes(ComponentBase): def _init_component(self): plugin_list = self._find_plugins() self._register_all_plugin_endpoints(plugin_list) def _register_all_plugin_endpoints(self, plugins_by_category): for plugin_type, plugin_list in plugins_by_category: for plugin in plugin_list: if self._module_has_routes(plugin, plugin_type): self._import_module_routes(plugin, plugin_type) def _find_plugins(self): plugin_list = [] for plugin_category in PLUGIN_CATEGORIES: plugin_list.append((plugin_category, self._get_modules_in_path('{}/{}'.format( PLUGIN_DIR, plugin_category)))) return plugin_list
GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. ''' import os from subprocess import CalledProcessError from tempfile import NamedTemporaryFile from common_helper_files import get_files_in_dir, get_dirs_in_dir from common_helper_process import execute_shell_command from helperFunctions.fileSystem import get_src_dir SIGNATURE_DIR = os.path.join(get_src_dir(), 'analysis/signatures') def _create_joint_signature_file(directory, tmp_file): all_signatures = list() for signature_file in sorted(get_files_in_dir(directory)): with open(signature_file, 'rb') as fd: all_signatures.append(fd.read()) with open(tmp_file.name, 'wb') as fd: fd.write(b'\x0a'.join(all_signatures)) def _get_plugin_name(plugin_path): return plugin_path.split('/')[-2]
def get_analysis_view(view_name): view_path = os.path.join( get_src_dir(), 'web_interface/templates/analysis_plugins/{}.html'.format(view_name)) return get_binary_from_file(view_path).decode('utf-8')
class AnalysisPlugin(AnalysisBasePlugin): ''' This plug-in tries to find and crack passwords ''' NAME = 'users_and_passwords' DEPENDENCIES = [] MIME_BLACKLIST = ['audio', 'filesystem', 'image', 'video'] DESCRIPTION = 'search for UNIX and httpd password files, parse them and try to crack the passwords' VERSION = '0.4.4' wordlist_path = os.path.join(get_src_dir(), 'bin/passwords.txt') def __init__(self, plugin_administrator, config=None, recursive=True): self.config = config super().__init__(plugin_administrator, config=config, recursive=recursive, no_multithread=True, plugin_path=__file__) def process_object(self, file_object): if self.NAME not in file_object.processed_analysis: file_object.processed_analysis[self.NAME] = {} file_object.processed_analysis[self.NAME]['summary'] = [] for passwd_regex in [ b'[a-zA-Z][a-zA-Z0-9_-]{2,15}:[^:]?:\\d+:\\d*:[^:]*:[^:]*:[^\n ]*', b'[a-zA-Z][a-zA-Z0-9_-]{2,15}:\\$[^\\$]+\\$[^\\$]+\\$[a-zA-Z0-9\\./+]{16,128}={0,3}' ]: passwd_entries = re.findall(passwd_regex, file_object.binary) if passwd_entries: result = self._generate_analysis_entry(passwd_entries) file_object.processed_analysis[self.NAME].update(result) file_object.processed_analysis[self.NAME]['summary'] += list( result.keys()) self._add_found_password_tag(file_object, result) return file_object def _add_found_password_tag(self, file_object, result): for password_entry in result: if 'password' in result[password_entry]: self.add_analysis_tag( file_object, '{}_{}'.format(password_entry, result[password_entry]['password']), 'Password: {}:{}'.format( password_entry, result[password_entry]['password']), TagColor.RED, True) def _generate_analysis_entry(self, passwd_entries): result = {} for entry in [e.split(b':') for e in passwd_entries]: key = entry[0].decode(encoding='utf_8', errors='replace') result[key] = { 'entry': b':'.join(entry).decode(encoding='utf_8', errors='replace') } try: if entry[1][0] == ord('$'): result[key]['password-hash'] = entry[1].decode( encoding='utf_8', errors='replace') cracked_pw = self._crack_hash(entry, result, key) result[key]['cracked'] = bool(cracked_pw) except (IndexError, AttributeError, TypeError): logging.error('Invalid Format:', exc_info=True) return result def _crack_hash(self, passwd_entry, result_dict, key): with NamedTemporaryFile() as fp: fp.write(b':'.join(passwd_entry[:2])) fp.seek(0) result_dict[key]['log'] = execute_shell_command( 'john --wordlist={} {}'.format(self.wordlist_path, fp.name)) output = execute_shell_command('john --show {}'.format( fp.name)).split('\n') if len(output) > 2: with suppress(KeyError): if '0 password hashes cracked' in output[-2]: result_dict[key]['ERROR'] = 'hash type is not supported' return False result_dict[key]['password'] = output[0].split(':')[1] return True return False