def _install_yara(distribution): # pylint: disable=too-complex logging.info('Installing yara') # CAUTION: Yara python binding is installed in install/common.py, because it is needed in the frontend as well. if distribution != 'fedora': apt_install_packages('bison', 'flex') if check_string_in_command_output('yara --version', '3.7.1'): logging.info('skipping yara installation (already installed)') return wget_output, wget_code = execute_shell_command_get_return_code( 'wget https://github.com/VirusTotal/yara/archive/v3.7.1.zip') if wget_code != 0: raise InstallationError(f'Error on yara download.\n{wget_output}') zip_output, return_code = execute_shell_command_get_return_code( 'unzip v3.7.1.zip') Path('v3.7.1.zip').unlink() if return_code != 0: raise InstallationError(f'Error on yara extraction.\n{zip_output}') yara_folder = [ child for child in Path('.').iterdir() if 'yara-3.' in child.name ][0] with OperateInDirectory(yara_folder.name, remove=True): os.chmod('bootstrap.sh', 0o775) for command in [ './bootstrap.sh', './configure --enable-magic', 'make -j$(nproc)', 'sudo make install' ]: output, return_code = execute_shell_command_get_return_code( command) if return_code != 0: raise InstallationError( f'Error in yara installation.\n{output}')
def main(distribution): logging.info('Setting up mongo database') if distribution == 'xenial': _add_mongo_mirror_to_sources() apt_update_sources() apt_install_packages('mongodb-org') else: apt_install_packages('mongodb') # creating DB directory fact_db_directory = _get_db_directory() mkdir_output, _ = execute_shell_command_get_return_code('sudo mkdir -p --mode=0744 {}'.format(fact_db_directory)) chown_output, chown_code = execute_shell_command_get_return_code('sudo chown {}:{} {}'.format(os.getuid(), os.getgid(), fact_db_directory)) if chown_code != 0: raise InstallationError('Failed to set up database directory. Check if parent folder exists\n{}'.format('\n'.join((mkdir_output, chown_output)))) # initializing DB authentication logging.info('Initialize database') with OperateInDirectory('..'): init_output, init_code = execute_shell_command_get_return_code('python3 init_database.py') if init_code != 0: raise InstallationError('Unable to initialize database\n{}'.format(init_output)) with OperateInDirectory('../../'): with suppress(FileNotFoundError): Path('start_fact_db').unlink() Path('start_fact_db').symlink_to('src/start_fact_db.py') return 0
def _install_yara(): logging.info('Installing yara') # CAUTION: Yara python binding is installed in bootstrap_common, because it is needed in the frontend as well. apt_install_packages('bison', 'flex', 'libmagic-dev') if check_string_in_command('yara --version', '3.7.1'): logging.info('skipping yara installation (already installed)') else: broken, output = False, '' wget_output, wget_code = execute_shell_command_get_return_code('wget https://github.com/VirusTotal/yara/archive/v3.7.1.zip') if wget_code != 0: raise InstallationError('Error on yara download.\n{}'.format(wget_output)) zip_output, zip_code = execute_shell_command_get_return_code('unzip v3.7.1.zip') if zip_code == 0: yara_folder = [child for child in Path('.').iterdir() if 'yara-3.' in child.name][0] with OperateInDirectory(yara_folder.name, remove=True): os.chmod('bootstrap.sh', 0o775) for command in ['./bootstrap.sh', './configure --enable-magic', 'make -j$(nproc)', 'sudo make install']: output, return_code = execute_shell_command_get_return_code(command) if return_code != 0: broken = True break else: raise InstallationError('Error on yara extraction.\n{}'.format(zip_output)) Path('v3.7.1.zip').unlink() if broken: raise InstallationError('Error in yara installation.\n{}'.format(output))
def _mount_from_boot_record(file_path, tmp_dir): output, return_code = execute_shell_command_get_return_code( 'sudo kpartx -a -v {}'.format(file_path)) sleep( 1 ) # Necessary since initialization of special devices seem to take some time if not return_code == 0: return 'Failed to mount master boot record image:\n{}'.format(output) loop_devices = _extract_loop_devices(output) with TemporaryDirectory() as mount_dir: for index, loop_device in enumerate(loop_devices): output += _process_loop_device(loop_device, mount_dir, tmp_dir, index) if loop_devices: # Bug in kpartx doesn't allow -d to work on long file names (as in /storage/path/<prefix>/<sha_hash>_<length>) # thus "host" loop device is used instead of filename k_output, return_code = execute_shell_command_get_return_code( 'sudo kpartx -d -v {}'.format(_get_host_loop(loop_devices))) execute_shell_command('sudo losetup -d {}'.format( _get_host_loop(loop_devices))) return output + k_output return output
def _mount_from_boot_record(file_path, tmp_dir): output, return_code = execute_shell_command_get_return_code( 'sudo kpartx -a -v {}'.format(file_path)) sleep( 1 ) # Necessary since initialization of special devices seem to take some time # kpartx may return an error on one partition but others are still loaded correctly. loop_devices = _extract_loop_devices(output) with TemporaryDirectory() as mount_dir: for index, loop_device in enumerate(loop_devices): output += _process_loop_device(loop_device, mount_dir, tmp_dir, index) if loop_devices: # Occasionally device mapping isn't removed correctly and results in losetup -d to fail, so remove explicitly for loop_dev in loop_devices: execute_shell_command( f'sudo dmsetup remove /dev/mapper/{loop_dev}') # Bug in kpartx doesn't allow -d to work on long file names (as in /storage/path/<prefix>/<sha_hash>_<length>) # thus "host" loop device is used instead of filename k_output, return_code = execute_shell_command_get_return_code( f'sudo kpartx -d -v {_get_host_loop(loop_devices)}') execute_shell_command( f'sudo losetup -d {_get_host_loop(loop_devices)}') return output + k_output return output
def _install_yara(): # pylint: disable=too-complex # CAUTION: Yara python binding is installed in install/common.py, because it is needed in the frontend as well. try: latest_url = requests.get('https://github.com/VirusTotal/yara/releases/latest').url latest_version = latest_url.split('/tag/')[1] except (AttributeError, KeyError): raise InstallationError('Could not find latest yara version') from None installed_version, return_code = execute_shell_command_get_return_code('yara --version') if return_code == 0 and installed_version.strip() == latest_version.strip('v'): logging.info('Skipping yara installation: Already installed and up to date') return logging.info(f'Installing yara {latest_version}') archive = f'{latest_version}.zip' download_url = f'https://github.com/VirusTotal/yara/archive/refs/tags/{archive}' wget_output, wget_code = execute_shell_command_get_return_code(f'wget {download_url}') if wget_code != 0: raise InstallationError(f'Error on yara download.\n{wget_output}') zip_output, return_code = execute_shell_command_get_return_code(f'unzip {archive}') Path(archive).unlink() if return_code != 0: raise InstallationError(f'Error on yara extraction.\n{zip_output}') yara_folder = [p for p in Path('.').iterdir() if p.name.startswith('yara-')][0] with OperateInDirectory(yara_folder.name, remove=True): os.chmod('bootstrap.sh', 0o775) for command in ['./bootstrap.sh', './configure --enable-magic', 'make -j$(nproc)', 'sudo make install']: output, return_code = execute_shell_command_get_return_code(command) if return_code != 0: raise InstallationError(f'Error in yara installation.\n{output}')
def _remove_folder(folder_name): try: shutil.rmtree(folder_name) except PermissionError: logging.debug('Falling back on root permission for deleting {}'.format(folder_name)) execute_shell_command_get_return_code('sudo rm -rf {}'.format(folder_name)) except Exception as exception: raise InstallationError(exception)
def main(): with OperateInDirectory(str(MIME_DIR)): cat_output, cat_code = execute_shell_command_get_return_code('cat custom_* > custommime') file_output, file_code = execute_shell_command_get_return_code('file -C -m custommime') mv_output, mv_code = execute_shell_command_get_return_code('mv -f custommime.mgc ../bin/') if any(code != 0 for code in (cat_code, file_code, mv_code)): exit('Failed to properly compile magic file\n{}'.format('\n'.join((cat_output, file_output, mv_output)))) Path('custommime').unlink()
def _add_mongo_mirror_to_sources(): apt_key_output, apt_key_code = execute_shell_command_get_return_code( 'sudo -E apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5') execute_shell_command_get_return_code('sudo rm /etc/apt/sources.list.d/mongodb-org-3.*') tee_output, tee_code = execute_shell_command_get_return_code( 'echo "deb https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list') if any(code != 0 for code in (apt_key_code, tee_code)): raise InstallationError('Unable to set up mongodb installation\n{}'.format('\n'.join((apt_key_output, tee_output))))
def main(distribution): # dependencies if distribution == 'fedora': dnf_install_packages('python-devel', 'python-setuptools') dnf_install_packages('libjpeg-devel') dnf_install_packages('openssl-devel', 'python3-tkinter') else: apt_install_packages('python-dev', 'python-setuptools') apt_install_packages('libjpeg-dev') apt_install_packages('libssl-dev', 'python3-tk') pip3_install_packages('pluginbase', 'Pillow', 'cryptography', 'pyopenssl', 'matplotlib', 'docker', 'networkx') # install yara _install_yara(distribution) # build extraction docker container logging.info('Building fact extraction container') output, return_code = execute_shell_command_get_return_code( 'docker pull fkiecad/fact_extractor') if return_code != 0: raise InstallationError( 'Failed to pull extraction container:\n{}'.format(output)) # installing common code modules pip3_install_packages( 'git+https://github.com/fkie-cad/common_helper_yara.git') pip3_install_packages( 'git+https://github.com/mass-project/common_analysis_base.git') # install plug-in dependencies _install_plugins(distribution) # configure environment _edit_sudoers() _edit_environment() # create directories _create_firmware_directory() # compiling yara signatures compile_signatures() _, yarac_return = execute_shell_command_get_return_code( 'yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc' ) if yarac_return != 0: raise InstallationError('Failed to compile yara test signatures') with OperateInDirectory('../../'): with suppress(FileNotFoundError): Path('start_fact_backend').unlink() Path('start_fact_backend').symlink_to('src/start_fact_backend.py') return 0
def _create_firmware_directory(): logging.info('Creating firmware directory') config = load_main_config() data_dir_name = config.get('data_storage', 'firmware_file_storage_directory') mkdir_output, mkdir_code = execute_shell_command_get_return_code('sudo mkdir -p --mode=0744 {}'.format(data_dir_name)) chown_output, chown_code = execute_shell_command_get_return_code('sudo chown {}:{} {}'.format(os.getuid(), os.getgid(), data_dir_name)) if not all(code == 0 for code in (mkdir_code, chown_code)): raise InstallationError('Failed to create directories for binary storage\n{}\n{}'.format(mkdir_output, chown_output))
def _edit_sudoers(): logging.info('add rules to sudo...') username = os.environ['USER'] sudoers_content = '\n'.join(('{}\tALL=NOPASSWD: {}'.format(username, command) for command in ('/bin/mount', '/bin/umount', '/bin/mknod', '/usr/local/bin/sasquatch', '/bin/rm', '/bin/cp', '/bin/dd', '/bin/chown'))) Path('/tmp/fact_overrides').write_text('{}\n'.format(sudoers_content)) chown_output, chown_code = execute_shell_command_get_return_code('sudo chown root:root /tmp/fact_overrides') mv_output, mv_code = execute_shell_command_get_return_code('sudo mv /tmp/fact_overrides /etc/sudoers.d/fact_overrides') if not chown_code == mv_code == 0: raise InstallationError('Editing sudoers file did not succeed\n{}\n{}'.format(chown_output, mv_output))
def _add_mongo_mirror(distribution): apt_key_output, apt_key_code = execute_shell_command_get_return_code( MONGO_MIRROR_COMMANDS[distribution]['key']) tee_output, tee_code = execute_shell_command_get_return_code( MONGO_MIRROR_COMMANDS[distribution]['sources']) if any(code != 0 for code in (apt_key_code, tee_code)): raise InstallationError( 'Unable to set up mongodb installation\n{}'.format('\n'.join( (apt_key_output, tee_output))))
def main(distribution): # dependencies apt_install_packages('python-dev', 'python-setuptools') apt_install_packages('libjpeg-dev', 'liblzma-dev', 'liblzo2-dev', 'zlib1g-dev') apt_install_packages('libssl-dev python3-tk') pip3_install_packages('pluginbase', 'Pillow', 'cryptography', 'pyopenssl', 'entropy', 'matplotlib') apt_install_packages('python-pip') # removes due to compatibilty reasons apt_remove_packages('python-lzma') pip2_remove_packages('pyliblzma') apt_install_packages('python-lzma') # install yara _install_yara() # installing unpacker _install_unpacker(distribution == 'xenial') # installing common code modules pip3_install_packages('git+https://github.com/fkie-cad/common_helper_process.git') pip3_install_packages('git+https://github.com/fkie-cad/common_helper_yara.git') pip3_install_packages('git+https://github.com/fkie-cad/common_helper_unpacking_classifier.git') pip3_install_packages('git+https://github.com/mass-project/common_analysis_base.git') # install plug-in dependencies _install_plugins() # compile custom magic file with OperateInDirectory('../mime'): cat_output, cat_code = execute_shell_command_get_return_code('cat custom_* > custommime') file_output, file_code = execute_shell_command_get_return_code('file -C -m custommime') mv_output, mv_code = execute_shell_command_get_return_code('mv -f custommime.mgc ../bin/') if any(code != 0 for code in (cat_code, file_code, mv_code)): raise InstallationError('Failed to properly compile magic file\n{}'.format('\n'.join((cat_output, file_output, mv_output)))) Path('custommime').unlink() # configure environment _edit_sudoers() _edit_environment() # create directories _create_firmware_directory() # compiling yara signatures compile_signatures() _, yarac_return = execute_shell_command_get_return_code('yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc') if yarac_return != 0: raise InstallationError('Failed to compile yara test signatures') with OperateInDirectory('../../'): with suppress(FileNotFoundError): Path('start_fact_backend').unlink() Path('start_fact_backend').symlink_to('src/start_fact_backend.py') return 0
def wget_static_web_content(url, target_folder, additional_actions, resource_logging_name=None): logging.info('Install static {} content'.format(resource_logging_name if resource_logging_name else url)) with OperateInDirectory(target_folder): wget_output, wget_code = execute_shell_command_get_return_code('wget -nc {}'.format(url)) if wget_code != 0: raise InstallationError('Failed to fetch resource at {}\n{}'.format(url, wget_output)) for action in additional_actions: action_output, action_code = execute_shell_command_get_return_code(action) if action_code != 0: raise InstallationError('Problem in processing resource at {}\n{}'.format(url, action_output))
def _install_stuffit(): logging.info('Installing stuffit') _, wget_code = execute_shell_command_get_return_code('wget -O - http://my.smithmicro.com/downloads/files/stuffit520.611linux-i386.tar.gz | tar -zxv') if wget_code == 0: _, cp_code = execute_shell_command_get_return_code('sudo cp bin/unstuff /usr/local/bin/') else: cp_code = 255 _, rm_code = execute_shell_command_get_return_code('rm -fr bin doc man') if not all(code == 0 for code in (wget_code, cp_code, rm_code)): raise InstallationError('Error in installation of unstuff')
def _install_plugins(): logging.info('Installing plugins') find_output, return_code = execute_shell_command_get_return_code('find ../plugins -iname "install.sh"') if return_code != 0: raise InstallationError('Error retrieving plugin installation scripts') for install_script in find_output.splitlines(keepends=False): logging.info('Running {}'.format(install_script)) shell_output, return_code = execute_shell_command_get_return_code(install_script) if return_code != 0: raise InstallationError('Error in installation of {} plugin\n{}'.format(Path(install_script).parent.name, shell_output))
def _change_owner_of_output_files(files_dir: Path, owner: str) -> int: if not match(r'\d+:\d+', owner): logging.error( 'ownership string should have the format <user id>:<group id>') return 1 _, return_code_chown = execute_shell_command_get_return_code( f'sudo chown -R {owner} {files_dir}') _, return_code_chmod = execute_shell_command_get_return_code( f'sudo chmod -R u+rw {files_dir}') return return_code_chmod | return_code_chown
def _update_submodules(): _, is_repository = execute_shell_command_get_return_code('git status') if is_repository == 0: git_output, git_code = execute_shell_command_get_return_code( '(cd ../../ && git submodule foreach "git pull")') if git_code != 0: raise InstallationError( 'Failed to update submodules\n{}'.format(git_output)) else: logging.warning( 'FACT is not set up using git. Note that *adding submodules* won\'t work!!' )
def main(radare, nginx): pip3_install_packages( 'werkzeug==0.16.1' ) # Multiple flask plugins break on werkzeug > 0.16.1 pip3_install_packages('flask', 'flask_restful', 'flask_security', 'flask_sqlalchemy', 'flask-paginate', 'Flask-API', 'uwsgi', 'bcrypt', 'python-dateutil', 'si-prefix', 'email-validator') # installing web/js-frameworks _install_css_and_js_files() # create user database _create_directory_for_authentication() if nginx: _install_nginx() if radare: logging.info('Initializing docker container for radare') execute_shell_command_get_return_code( 'virtualenv {}'.format(COMPOSE_VENV)) output, return_code = execute_shell_command_get_return_code( '{} install -U docker-compose'.format(COMPOSE_VENV / 'bin' / 'pip')) if return_code != 0: raise InstallationError( 'Failed to set up virtualenv for docker-compose\n{}'.format( output)) with OperateInDirectory('radare'): output, return_code = execute_shell_command_get_return_code( '{} build'.format(COMPOSE_VENV / 'bin' / 'docker-compose')) if return_code != 0: raise InstallationError( 'Failed to initialize radare container:\n{}'.format( output)) # pull pdf report container logging.info('Pulling pdf report container') output, return_code = execute_shell_command_get_return_code( 'docker pull fkiecad/fact_pdf_report') if return_code != 0: raise InstallationError( 'Failed to pull pdf report container:\n{}'.format(output)) with OperateInDirectory('../../'): with suppress(FileNotFoundError): Path('start_fact_frontend').unlink() Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py') return 0
def _create_directory_for_authentication(): # pylint: disable=invalid-name logging.info('Creating directory for authentication') config = load_main_config() dburi = config.get('data_storage', 'user_database') factauthdir = '/'.join(dburi.split('/')[:-1])[10:] # FIXME this should be beautified with pathlib mkdir_output, mkdir_code = execute_shell_command_get_return_code('sudo mkdir -p --mode=0744 {}'.format(factauthdir)) chown_output, chown_code = execute_shell_command_get_return_code('sudo chown {}:{} {}'.format(os.getuid(), os.getgid(), factauthdir)) if not all(return_code == 0 for return_code in [mkdir_code, chown_code]): raise InstallationError('Error in creating directory for authentication database.\n{}'.format('\n'.join((mkdir_output, chown_output))))
def test_start_script_help_and_version(script): output, return_code = execute_shell_command_get_return_code('{} -h'.format( os.path.join(get_src_dir(), script)), timeout=5) assert return_code == 0 assert 'usage: {}'.format(script) in output output, return_code = execute_shell_command_get_return_code('{} -V'.format( os.path.join(get_src_dir(), script)), timeout=5) assert output[0:5] == 'FACT ' assert return_code == 0 gc.collect()
def test_start_script_help_and_version(script, expected_str): output, return_code = execute_shell_command_get_return_code('{} -h'.format( os.path.join(get_src_dir(), script)), timeout=5) assert return_code == 0 assert 'usage: {}'.format(script) in output output, return_code = execute_shell_command_get_return_code('{} -V'.format( os.path.join(get_src_dir(), script)), timeout=5) assert expected_str in output, 'Wrong output {}'.format(output) assert return_code == 0 gc.collect()
def execute_commands_and_raise_on_return_code(commands, error=None): for command in commands: bad_return = error if error else 'execute {}'.format(command) output, return_code = execute_shell_command_get_return_code(command) if return_code != 0: raise InstallationError('Failed to {}\n{}'.format( bad_return, output))
def build_pdf_report(firmware: Firmware, folder: Path) -> Path: ''' Creates a pdf report for the given firmware by calling the fact_pdf_report docker container. .. admonition:: About the pdf report The pdf report tool is based on the jinja2 templating engine and renders a latex file that is build into a one page overview of the analysis results. For all technical details refer to the `implementation <https://github.com/fkie-cad/fact_pdf_report>`_. :param firmware: The firmware to generate the pdf report for :param folder: An empty folder in which to generate the resulting pdf in :return: The path to the generated pdf file inside the given folder ''' _initialize_subfolder(folder, firmware) output, return_code = execute_shell_command_get_return_code( 'docker run -m 512m -v {}:/tmp/interface --rm fkiecad/fact_pdf_report'. format(folder)) if return_code != 0: logging.error( 'Failed to execute pdf generator with code {}:\n{}'.format( return_code, output)) raise RuntimeError('Could not create PDF report') _claim_folder_contents(folder) pdf_path = _find_pdf(folder) return pdf_path
def remove_folder(folder_name: str): ''' Python equivalent to `rm -rf`. Remove a directory an all included files. If administrative rights are necessary, this effectively falls back to `sudo rm -rf`. :param folder_name: Path to directory to remove. ''' try: shutil.rmtree(folder_name) except PermissionError: logging.debug('Falling back on root permission for deleting {}'.format( folder_name)) execute_shell_command_get_return_code( 'sudo rm -rf {}'.format(folder_name)) except Exception as exception: raise InstallationError(exception) from None
def install_github_project(project_path: str, commands: List[str]): ''' Install github project by cloning it, running a set of commands and removing the cloned files afterwards. :param project_path: Github path to project. For FACT this is 'fkie-cad/FACT_core'. :param commands: List of commands to run after cloning to install project. :Example: .. code-block:: python install_github_project( 'ghusername/c-style-project', ['./configure', 'make', 'sudo make install'] ) ''' log_current_packages((project_path, )) folder_name = Path(project_path).name _checkout_github_project(project_path, folder_name) with OperateInDirectory(folder_name, remove=True): error = None for command in commands: output, return_code = execute_shell_command_get_return_code( command) if return_code != 0: error = 'Error while processing github project {}!\n{}'.format( project_path, output) break if error: raise InstallationError(error)
def _install_nginx(): apt_install_packages('nginx') _generate_and_install_certificate() _configure_nginx() nginx_output, nginx_code = execute_shell_command_get_return_code('sudo nginx -s reload') if nginx_code != 0: raise InstallationError('Failed to start nginx\n{}'.format(nginx_output))
def _get_db_directory(): output, return_code = execute_shell_command_get_return_code( r'grep -oP "dbPath:[\s]*\K[^\s]+" ../config/mongod.conf') if return_code != 0: raise InstallationError( 'Unable to locate target for database directory') return output.strip()
def _create_variety_data(config): varietyjs_script_path = Path( get_src_dir()) / config['data_storage']['variety_path'] mongo_call = ( 'mongo --port {mongo_port} -u "{username}" -p "{password}" --authenticationDatabase "admin" ' .format( mongo_port=config['data_storage']['mongo_port'], username=config['data_storage']['db_admin_user'], password=config['data_storage']['db_admin_pw'], )) output, return_code = execute_shell_command_get_return_code( '{mongo_call} {database} --eval "var collection = \'file_objects\', persistResults=true" {script_path}' .format(mongo_call=mongo_call, database=config['data_storage']['main_database'], script_path=varietyjs_script_path), timeout=None) if return_code == 0: execute_shell_command( '{mongo_call} varietyResults --eval \'{command}\''.format( mongo_call=mongo_call, command= 'db.file_objectsKeys.deleteMany({"_id.key": {"$regex": "skipped|file_system_flag"}})' ), ) logging.debug(output) return return_code