def test_operate_in_directory():
'''
TempDir structure:
├ file1
└ folder
└ file2
'''
with TemporaryDirectory('fact_test') as tmp_dir:
tmp_path = Path(tmp_dir)
folder = tmp_path / 'folder'
folder.mkdir()
file1 = tmp_path / 'file1'
file1.touch()
file2 = folder / 'file2'
file2.touch()
assert not (Path(file1.name).is_file() or Path(file2.name).is_file()
or Path(folder.name).is_dir())
current_dir = os.getcwd()
with OperateInDirectory(tmp_dir):
assert Path(file1.name).is_file()
assert current_dir != os.getcwd()
assert current_dir == os.getcwd()
with OperateInDirectory(folder, remove=True):
assert Path(file2.name).is_file()
assert file1.is_file() and not file2.is_file() and not folder.is_dir()
def main(distribution):
logging.info('Setting up mongo database')
if distribution == 'xenial':
_add_mongo_mirror_to_sources()
apt_update_sources()
apt_install_packages('mongodb-org')
else:
apt_install_packages('mongodb')
# creating DB directory
fact_db_directory = _get_db_directory()
mkdir_output, _ = execute_shell_command_get_return_code('sudo mkdir -p --mode=0744 {}'.format(fact_db_directory))
chown_output, chown_code = execute_shell_command_get_return_code('sudo chown {}:{} {}'.format(os.getuid(), os.getgid(), fact_db_directory))
if chown_code != 0:
raise InstallationError('Failed to set up database directory. Check if parent folder exists\n{}'.format('\n'.join((mkdir_output, chown_output))))
# initializing DB authentication
logging.info('Initialize database')
with OperateInDirectory('..'):
init_output, init_code = execute_shell_command_get_return_code('python3 init_database.py')
if init_code != 0:
raise InstallationError('Unable to initialize database\n{}'.format(init_output))
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_db').unlink()
Path('start_fact_db').symlink_to('src/start_fact_db.py')
return 0
def main(distribution):
# dependencies
apt_install_packages('python-dev', 'python-setuptools')
apt_install_packages('libjpeg-dev', 'liblzma-dev', 'liblzo2-dev', 'zlib1g-dev')
apt_install_packages('libssl-dev python3-tk')
pip3_install_packages('pluginbase', 'Pillow', 'cryptography', 'pyopenssl', 'entropy', 'matplotlib')
apt_install_packages('python-pip')
# removes due to compatibilty reasons
apt_remove_packages('python-lzma')
pip2_remove_packages('pyliblzma')
apt_install_packages('python-lzma')
# install yara
_install_yara()
# installing unpacker
_install_unpacker(distribution == 'xenial')
# installing common code modules
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_process.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_yara.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_unpacking_classifier.git')
pip3_install_packages('git+https://github.com/mass-project/common_analysis_base.git')
# install plug-in dependencies
_install_plugins()
# compile custom magic file
with OperateInDirectory('../mime'):
cat_output, cat_code = execute_shell_command_get_return_code('cat custom_* > custommime')
file_output, file_code = execute_shell_command_get_return_code('file -C -m custommime')
mv_output, mv_code = execute_shell_command_get_return_code('mv -f custommime.mgc ../bin/')
if any(code != 0 for code in (cat_code, file_code, mv_code)):
raise InstallationError('Failed to properly compile magic file\n{}'.format('\n'.join((cat_output, file_output, mv_output))))
Path('custommime').unlink()
# configure environment
_edit_sudoers()
_edit_environment()
# create directories
_create_firmware_directory()
# compiling yara signatures
compile_signatures()
_, yarac_return = execute_shell_command_get_return_code('yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc')
if yarac_return != 0:
raise InstallationError('Failed to compile yara test signatures')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_backend').unlink()
Path('start_fact_backend').symlink_to('src/start_fact_backend.py')
return 0
def main(radare, nginx):
pip3_install_packages(
'werkzeug==0.16.1'
) # Multiple flask plugins break on werkzeug > 0.16.1
pip3_install_packages('flask', 'flask_restful', 'flask_security',
'flask_sqlalchemy', 'flask-paginate', 'Flask-API',
'uwsgi', 'bcrypt', 'python-dateutil', 'si-prefix',
'email-validator')
# installing web/js-frameworks
_install_css_and_js_files()
# create user database
_create_directory_for_authentication()
if nginx:
_install_nginx()
if radare:
logging.info('Initializing docker container for radare')
execute_shell_command_get_return_code(
'virtualenv {}'.format(COMPOSE_VENV))
output, return_code = execute_shell_command_get_return_code(
'{} install -U docker-compose'.format(COMPOSE_VENV / 'bin' /
'pip'))
if return_code != 0:
raise InstallationError(
'Failed to set up virtualenv for docker-compose\n{}'.format(
output))
with OperateInDirectory('radare'):
output, return_code = execute_shell_command_get_return_code(
'{} build'.format(COMPOSE_VENV / 'bin' / 'docker-compose'))
if return_code != 0:
raise InstallationError(
'Failed to initialize radare container:\n{}'.format(
output))
# pull pdf report container
logging.info('Pulling pdf report container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_pdf_report')
if return_code != 0:
raise InstallationError(
'Failed to pull pdf report container:\n{}'.format(output))
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_frontend').unlink()
Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py')
return 0
def _install_yara():
logging.info('Installing yara')
# CAUTION: Yara python binding is installed in bootstrap_common, because it is needed in the frontend as well.
apt_install_packages('bison', 'flex', 'libmagic-dev')
if check_string_in_command('yara --version', '3.7.1'):
logging.info('skipping yara installation (already installed)')
else:
broken, output = False, ''
wget_output, wget_code = execute_shell_command_get_return_code('wget https://github.com/VirusTotal/yara/archive/v3.7.1.zip')
if wget_code != 0:
raise InstallationError('Error on yara download.\n{}'.format(wget_output))
zip_output, zip_code = execute_shell_command_get_return_code('unzip v3.7.1.zip')
if zip_code == 0:
yara_folder = [child for child in Path('.').iterdir() if 'yara-3.' in child.name][0]
with OperateInDirectory(yara_folder.name, remove=True):
os.chmod('bootstrap.sh', 0o775)
for command in ['./bootstrap.sh', './configure --enable-magic', 'make -j$(nproc)', 'sudo make install']:
output, return_code = execute_shell_command_get_return_code(command)
if return_code != 0:
broken = True
break
else:
raise InstallationError('Error on yara extraction.\n{}'.format(zip_output))
Path('v3.7.1.zip').unlink()
if broken:
raise InstallationError('Error in yara installation.\n{}'.format(output))
def install():
check_python_version()
args = _setup_argparser()
_setup_logging(args.log_level, args.log_file, debug_flag=args.debug)
welcome()
distribution = check_distribution()
none_chosen = not (args.frontend or args.db or args.backend)
installation_directory = get_directory_of_current_file() / 'install'
with OperateInDirectory(str(installation_directory)):
common(distribution)
if args.frontend or none_chosen:
frontend(not args.no_radare, args.nginx)
if args.db or none_chosen:
db(distribution)
if args.backend or none_chosen:
backend(distribution)
if args.statistic_cronjob:
install_statistic_cronjob()
logging.info('installation complete')
logging.warning('If FACT does not start, reload the environment variables with: source /etc/profile')
sys.exit()
def _install_yara(distribution): # pylint: disable=too-complex
logging.info('Installing yara')
# CAUTION: Yara python binding is installed in install/common.py, because it is needed in the frontend as well.
if distribution != 'fedora':
apt_install_packages('bison', 'flex')
if check_string_in_command_output('yara --version', '3.7.1'):
logging.info('skipping yara installation (already installed)')
return
wget_output, wget_code = execute_shell_command_get_return_code(
'wget https://github.com/VirusTotal/yara/archive/v3.7.1.zip')
if wget_code != 0:
raise InstallationError(f'Error on yara download.\n{wget_output}')
zip_output, return_code = execute_shell_command_get_return_code(
'unzip v3.7.1.zip')
Path('v3.7.1.zip').unlink()
if return_code != 0:
raise InstallationError(f'Error on yara extraction.\n{zip_output}')
yara_folder = [
child for child in Path('.').iterdir() if 'yara-3.' in child.name
][0]
with OperateInDirectory(yara_folder.name, remove=True):
os.chmod('bootstrap.sh', 0o775)
for command in [
'./bootstrap.sh', './configure --enable-magic',
'make -j$(nproc)', 'sudo make install'
]:
output, return_code = execute_shell_command_get_return_code(
command)
if return_code != 0:
raise InstallationError(
f'Error in yara installation.\n{output}')
def _install_yara(): # pylint: disable=too-complex
# CAUTION: Yara python binding is installed in install/common.py, because it is needed in the frontend as well.
try:
latest_url = requests.get('https://github.com/VirusTotal/yara/releases/latest').url
latest_version = latest_url.split('/tag/')[1]
except (AttributeError, KeyError):
raise InstallationError('Could not find latest yara version') from None
installed_version, return_code = execute_shell_command_get_return_code('yara --version')
if return_code == 0 and installed_version.strip() == latest_version.strip('v'):
logging.info('Skipping yara installation: Already installed and up to date')
return
logging.info(f'Installing yara {latest_version}')
archive = f'{latest_version}.zip'
download_url = f'https://github.com/VirusTotal/yara/archive/refs/tags/{archive}'
wget_output, wget_code = execute_shell_command_get_return_code(f'wget {download_url}')
if wget_code != 0:
raise InstallationError(f'Error on yara download.\n{wget_output}')
zip_output, return_code = execute_shell_command_get_return_code(f'unzip {archive}')
Path(archive).unlink()
if return_code != 0:
raise InstallationError(f'Error on yara extraction.\n{zip_output}')
yara_folder = [p for p in Path('.').iterdir() if p.name.startswith('yara-')][0]
with OperateInDirectory(yara_folder.name, remove=True):
os.chmod('bootstrap.sh', 0o775)
for command in ['./bootstrap.sh', './configure --enable-magic', 'make -j$(nproc)', 'sudo make install']:
output, return_code = execute_shell_command_get_return_code(command)
if return_code != 0:
raise InstallationError(f'Error in yara installation.\n{output}')
def install():
check_python_version()
args = _setup_argparser()
_setup_logging(args.log_level, args.log_file, debug_flag=args.debug)
welcome()
distribution = check_distribution()
none_chosen = not (args.frontend or args.db or args.backend)
# TODO maybe replace this with an cli argument
skip_docker = FACT_INSTALLER_SKIP_DOCKER is not None
# Note that the skip_docker environment variable overrides the cli argument
only_docker = not skip_docker and none_chosen and (
args.backend_docker_images or args.frontend_docker_images)
installation_directory = get_directory_of_current_file() / 'install'
with OperateInDirectory(str(installation_directory)):
if not only_docker:
install_fact_components(args, distribution, none_chosen,
skip_docker)
else:
install_docker_images(args)
if args.statistic_cronjob:
install_statistic_cronjob()
logging.info('installation complete')
logging.warning(
'If FACT does not start, reload the environment variables with: source /etc/profile'
)
sys.exit(0)
def _install_and_patch_bootstrap():
with OperateInDirectory('../web_interface/static'):
wget_static_web_content(
'https://github.com/twbs/bootstrap/releases/download/v3.3.7/bootstrap-3.3.7-dist.zip',
'.', [
'unzip -o bootstrap-3.3.7-dist.zip',
'rm bootstrap-3.3.7-dist.zip', 'rm -rf bootstrap',
'mv bootstrap-3.3.7-dist bootstrap'
], 'bootstrap')
_patch_bootstrap()
wget_static_web_content(
'https://ajax.googleapis.com/ajax/libs/jquery/1.12.0/jquery.min.js',
'bootstrap/js', [], 'jquery')
wget_static_web_content(
'https://raw.githubusercontent.com/Eonasdan/bootstrap-datetimepicker/master/build/js/bootstrap-datetimepicker.min.js',
'bootstrap/js', [], 'datetimepicker js')
wget_static_web_content(
'https://raw.githubusercontent.com/Eonasdan/bootstrap-datetimepicker/master/build/css/bootstrap-datetimepicker.min.css',
'bootstrap/css', [], 'datetimepicker css')
wget_static_web_content(
'https://raw.githubusercontent.com/moment/moment/develop/moment.js',
'bootstrap/js', [], 'moment.js')
if not Path('bootstrap3-editable').exists():
wget_static_web_content(
'https://vitalets.github.io/x-editable/assets/zip/bootstrap3-editable-1.5.1.zip',
'.', [
'unzip -o bootstrap3-editable-1.5.1.zip',
'rm bootstrap3-editable-1.5.1.zip CHANGELOG.txt LICENSE-MIT README.md',
'rm -rf inputs-ext'
], 'x-editable')
def main():
with OperateInDirectory(str(MIME_DIR)):
cat_output, cat_code = execute_shell_command_get_return_code('cat custom_* > custommime')
file_output, file_code = execute_shell_command_get_return_code('file -C -m custommime')
mv_output, mv_code = execute_shell_command_get_return_code('mv -f custommime.mgc ../bin/')
if any(code != 0 for code in (cat_code, file_code, mv_code)):
exit('Failed to properly compile magic file\n{}'.format('\n'.join((cat_output, file_output, mv_output))))
Path('custommime').unlink()
def main(distribution):
# dependencies
if distribution == 'fedora':
dnf_install_packages('python-devel', 'python-setuptools')
dnf_install_packages('libjpeg-devel')
dnf_install_packages('openssl-devel', 'python3-tkinter')
else:
apt_install_packages('python-dev', 'python-setuptools')
apt_install_packages('libjpeg-dev')
apt_install_packages('libssl-dev', 'python3-tk')
pip3_install_packages('pluginbase', 'Pillow', 'cryptography', 'pyopenssl',
'matplotlib', 'docker', 'networkx')
# install yara
_install_yara(distribution)
# build extraction docker container
logging.info('Building fact extraction container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_extractor')
if return_code != 0:
raise InstallationError(
'Failed to pull extraction container:\n{}'.format(output))
# installing common code modules
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_yara.git')
pip3_install_packages(
'git+https://github.com/mass-project/common_analysis_base.git')
# install plug-in dependencies
_install_plugins(distribution)
# configure environment
_edit_sudoers()
_edit_environment()
# create directories
_create_firmware_directory()
# compiling yara signatures
compile_signatures()
_, yarac_return = execute_shell_command_get_return_code(
'yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc'
)
if yarac_return != 0:
raise InstallationError('Failed to compile yara test signatures')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_backend').unlink()
Path('start_fact_backend').symlink_to('src/start_fact_backend.py')
return 0
def _install_css_and_js_files():
with OperateInDirectory('../web_interface/static'):
os.makedirs('web_css', exist_ok=True)
os.makedirs('web_js', exist_ok=True)
wget_static_web_content(
'https://github.com/vakata/jstree/zipball/3.3.9', '.', [
'unzip 3.3.9', 'rm 3.3.9', 'rm -rf ./web_js/jstree/vakata*',
'mv vakata* web_js/jstree'
], 'jstree')
wget_static_web_content(
'https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular.min.js',
'.', [], 'angularJS')
wget_static_web_content(
'https://github.com/chartjs/Chart.js/releases/download/v2.3.0/Chart.js',
'.', [], 'charts.js')
wget_static_web_content(
'https://registry.npmjs.org/vis-network/-/vis-network-8.5.2.tgz',
'.', [
'tar xf vis-network-8.5.2.tgz', 'rm -rf vis',
'mv package/dist vis'
], 'vis-network')
_build_highlight_js()
for css_url in [
'https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css',
'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.8.0/css/bootstrap-datepicker.standalone.css'
]:
wget_static_web_content(css_url, 'web_css', [])
for js_url in [
'https://cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min.js',
'https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js',
'https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js',
'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.8.0/js/bootstrap-datepicker.js',
'https://raw.githubusercontent.com/moment/moment/develop/moment.js'
]:
wget_static_web_content(js_url, 'web_js', [])
if not Path('web_css/fontawesome').exists():
wget_static_web_content(
'https://use.fontawesome.com/releases/v5.13.0/fontawesome-free-5.13.0-web.zip',
'.', [
'unzip fontawesome-free-5.13.0-web.zip',
'rm fontawesome-free-5.13.0-web.zip',
'mv fontawesome-free-5.13.0-web web_css/fontawesome'
])
if not Path('bootstrap3-editable').exists():
wget_static_web_content(
'https://vitalets.github.io/x-editable/assets/zip/bootstrap3-editable-1.5.1.zip',
'.', [
'unzip -o bootstrap3-editable-1.5.1.zip',
'rm bootstrap3-editable-1.5.1.zip CHANGELOG.txt LICENSE-MIT README.md',
'rm -rf inputs-ext'
], 'x-editable')
def wget_static_web_content(url, target_folder, additional_actions, resource_logging_name=None):
logging.info('Install static {} content'.format(resource_logging_name if resource_logging_name else url))
with OperateInDirectory(target_folder):
wget_output, wget_code = execute_shell_command_get_return_code('wget -nc {}'.format(url))
if wget_code != 0:
raise InstallationError('Failed to fetch resource at {}\n{}'.format(url, wget_output))
for action in additional_actions:
action_output, action_code = execute_shell_command_get_return_code(action)
if action_code != 0:
raise InstallationError('Problem in processing resource at {}\n{}'.format(url, action_output))
def build(self):
url_binwalk = f'https://github.com/ReFirmLabs/binwalk/archive/refs/tags/v{BINWALK_VERSION}.tar.gz'
dest_binwalk = f'binwalk-v{BINWALK_VERSION}.tar.gz'
urllib.request.urlretrieve(url_binwalk, dest_binwalk)
run_cmd_with_logging(f'tar -xf {dest_binwalk}')
with OperateInDirectory(f'binwalk-{BINWALK_VERSION}'):
if is_virtualenv():
run_cmd_with_logging('pip install -U .')
else:
run_cmd_with_logging('sudo -EH pip3 install -U .')
def _build_highlight_js():
logging.info('Installing highlight js')
highlight_js_url = 'https://highlightjs.org/download/'
highlight_js_dir = 'highlight.js'
highlight_js_zip = 'highlight.js.zip'
if Path(highlight_js_dir).is_dir():
OperateInDirectory._remove_folder(highlight_js_dir)
req = requests.get('https://highlightjs.org/download/')
crsf_cookie = req.headers['Set-Cookie']
csrf_token = crsf_cookie.split(';')[0].split('=')[1]
commands = [
'wget {} --header="Host: highlightjs.org" --header="User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0" --header="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" --header="Accept-Language: en-GB,en;q=0.5" --header="Accept-Encoding: gzip, deflate, br" --header="Referer: https://highlightjs.org/download/" --header="Content-Type: application/x-www-form-urlencoded" --header="Cookie: csrftoken={}" --header="DNT: 1" --header="Connection: keep-alive" --header="Upgrade-Insecure-Requests: 1" --post-data="apache.js=on&bash.js=on&coffeescript.js=on&cpp.js=on&cs.js=on&csrfmiddlewaretoken={}&css.js=on&diff.js=on&http.js=on&ini.js=on&java.js=on&javascript.js=on&json.js=on&makefile.js=on&markdown.js=on&nginx.js=on&objectivec.js=on&perl.js=on&php.js=on&python.js=on&ruby.js=on&shell.js=on&sql.js=on&xml.js=on" -O {}'
.format(highlight_js_url, csrf_token, csrf_token, highlight_js_zip),
'unzip {} -d {}'.format(highlight_js_zip, highlight_js_dir)
]
execute_commands_and_raise_on_return_code(
commands, error='Failed to set up highlight.js')
Path(highlight_js_zip).unlink()
def main():
check_python_version()
args = _setup_argparser()
_setup_logging(debug_flag=args.debug)
distribution = check_distribution()
logging.info('{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
installation_directory = str(Path(__file__).parent / 'install')
with OperateInDirectory(installation_directory):
common(distribution)
unpacker(distribution)
logging.info('installation complete')
def _install_freetz():
logging.info('Installing FREETZ')
current_user = getuser()
with TemporaryDirectory(prefix='fact_freetz') as build_directory:
with OperateInDirectory(build_directory):
os.umask(0o022)
install_github_project('Freetz/freetz', [
'sudo useradd -M makeuser',
'sudo ln -s $(which python3) ./python',
f'sudo chown -R makeuser {build_directory}',
'sudo su makeuser -c "export PATH=$(pwd):$PATH && umask 0022 && make -j$(nproc) tools"',
f'sudo chmod -R 777 {build_directory}',
f'sudo chown -R {current_user} {build_directory}',
'cp tools/find-squashfs tools/unpack-kernel tools/freetz_bin_functions tools/unlzma tools/sfk '
f'tools/unsquashfs4-avm-be tools/unsquashfs4-avm-le tools/unsquashfs3-multi {BIN_DIR}',
'sudo userdel makeuser'
])
def _install_css_and_js_files():
with OperateInDirectory('../web_interface/static'):
os.makedirs('web_css', exist_ok=True)
os.makedirs('web_js', exist_ok=True)
wget_static_web_content(
'https://github.com/vakata/jstree/zipball/3.3.9', '.', [
'unzip 3.3.9', 'rm 3.3.9', 'rm -rf ./web_js/jstree/vakata*',
'mv vakata* web_js/jstree'
], 'jstree')
wget_static_web_content(
'https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular.min.js',
'.', [], 'angularJS')
wget_static_web_content(
'https://github.com/chartjs/Chart.js/releases/download/v2.3.0/Chart.js',
'.', [], 'charts.js')
_build_highlight_js()
for css_url in [
'https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css',
'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.8.0/css/bootstrap-datepicker.standalone.css',
'https://unpkg.com/[email protected]/styles/vis-network.min.css',
'https://cdn.jsdelivr.net/npm/diff2html/bundles/css/diff2html.min.css',
]:
wget_static_web_content(css_url, 'web_css', [])
for js_url in [
'https://cdnjs.cloudflare.com/ajax/libs/jquery/1.12.1/jquery.min.js',
'https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js',
'https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js',
'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.8.0/js/bootstrap-datepicker.js',
'https://raw.githubusercontent.com/moment/moment/develop/moment.js',
'https://unpkg.com/[email protected]/standalone/umd/vis-network.min.js',
'https://cdn.jsdelivr.net/npm/diff2html/bundles/js/diff2html-ui.min.js',
]:
wget_static_web_content(js_url, 'web_js', [])
if not Path('web_css/fontawesome').exists():
wget_static_web_content(
'https://use.fontawesome.com/releases/v5.13.0/fontawesome-free-5.13.0-web.zip',
'.', [
'unzip fontawesome-free-5.13.0-web.zip',
'rm fontawesome-free-5.13.0-web.zip',
'mv fontawesome-free-5.13.0-web web_css/fontawesome'
])
def _patch_bootstrap():
with OperateInDirectory('bootstrap/css'):
for file_name in [
'bootstrap.min.css', 'bootstrap.min.css.map',
'bootstrap-theme.min.css', 'bootstrap-theme.min.css.map',
'bootstrap.css.map', 'bootstrap-theme.css.map'
]:
Path(file_name).unlink()
_, first_code = execute_shell_command_get_return_code(
'patch --forward -r - bootstrap.css ../../../../install/patches/bootstrap.patch'
)
_, second_code = execute_shell_command_get_return_code(
'patch --forward -r - bootstrap-theme.css ../../../../install/patches/bootstrap-theme.patch'
)
if not first_code == second_code == 0:
raise InstallationError('Failed to patch bootstrap files')
def _install_docker_images(radare):
if radare:
logging.info('Initializing docker container for radare')
with OperateInDirectory('radare'):
output, return_code = execute_shell_command_get_return_code(
'docker-compose build')
if return_code != 0:
raise InstallationError(
'Failed to initialize radare container:\n{}'.format(
output))
# pull pdf report container
logging.info('Pulling pdf report container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_pdf_report')
if return_code != 0:
raise InstallationError(
'Failed to pull pdf report container:\n{}'.format(output))
def main(skip_docker, distribution):
apt_packages_path = INSTALL_DIR / 'apt-pkgs-backend.txt'
dnf_packages_path = INSTALL_DIR / 'dnf-pkgs-backend.txt'
if distribution != 'fedora':
pkgs = read_package_list_from_file(apt_packages_path)
apt_install_packages(*pkgs)
else:
pkgs = read_package_list_from_file(dnf_packages_path)
dnf_install_packages(*pkgs)
install_pip_packages(PIP_DEPENDENCIES)
# install yara
_install_yara()
_install_checksec()
if not skip_docker:
_install_docker_images()
# install plug-in dependencies
_install_plugins(distribution, skip_docker)
# configure environment
_edit_environment()
# create directories
_create_firmware_directory()
# compiling yara signatures
compile_signatures()
_, yarac_return = execute_shell_command_get_return_code('yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc')
if yarac_return != 0:
raise InstallationError('Failed to compile yara test signatures')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_backend').unlink()
Path('start_fact_backend').symlink_to('src/start_fact_backend.py')
return 0
def main(distribution): # pylint: disable=too-many-statements
_update_package_sources(distribution)
_update_submodules()
BIN_DIR.mkdir(exist_ok=True)
apt_packages_path = INSTALL_DIR / 'apt-pkgs-common.txt'
dnf_packages_path = INSTALL_DIR / 'dnf-pkgs-common.txt'
if distribution != 'fedora':
pkgs = read_package_list_from_file(apt_packages_path)
apt_install_packages(*pkgs)
else:
pkgs = read_package_list_from_file(dnf_packages_path)
dnf_install_packages(*pkgs)
if not is_virtualenv():
install_pip()
elif distribution != 'fedora':
run_cmd_with_logging('pip install -U pip setuptools wheel')
else:
# on fedora, extra setuptools will break some system tools like selinux ones
run_cmd_with_logging('pip install -U pip wheel')
install_pip_packages(PIP_DEPENDENCIES)
# VarietyJS (is executed by update_statistic.py)
if (BIN_DIR / 'spec').exists():
logging.warning('variety spec not overwritten')
else:
install_github_project('variety/variety', [
'git checkout 2f4d815', 'mv -f variety.js ../../bin/',
'mv -f spec ../../bin/'
])
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to(
'src/start_fact.py')
return 0
def install_files(self):
with TemporaryDirectory(dir=str(self.base_path)) as tmp_dir:
# We download a specific version of the package so no need to
# update downloaded files
if (Path(
f'{self.base_path}/test/data/test_tmp_dir/lib/libc.so.6'
).exists() and Path(
f'{self.base_path}/test/data/test_tmp_dir/lib/ld.so.1'
).exists() and Path(
f'{self.base_path}/test/data/test_tmp_dir_2/lib/libc.so.6'
).exists() and Path(
f'{self.base_path}/test/data/test_tmp_dir_2/lib/ld.so.1').
exists()):
return
url_libc6_mips = 'http://de.archive.ubuntu.com/ubuntu/pool/universe/c/cross-toolchain-base-ports/libc6-mips-cross_2.23-0ubuntu3cross1_all.deb'
dest_libc6_mips = f'{tmp_dir}/libc6-mips-cross_2.23-0ubuntu3cross1_all.deb'
urllib.request.urlretrieve(url_libc6_mips, dest_libc6_mips)
# We can't use `ar --output` because it was added in 2.34 but
# debian buster uses 2.31
with OperateInDirectory(tmp_dir):
run_cmd_with_logging(f'ar x {dest_libc6_mips} data.tar.xz')
run_cmd_with_logging(f'tar -xf {tmp_dir}/data.tar.xz -C {tmp_dir}')
Path('test/data/test_tmp_dir/lib').mkdir(exist_ok=True,
parents=True)
Path('test/data/test_tmp_dir_2/fact_extracted/lib').mkdir(
exist_ok=True, parents=True)
run_cmd_with_logging(
f'cp {tmp_dir}/usr/mips-linux-gnu/lib/libc-2.23.so test/data/test_tmp_dir/lib/libc.so.6'
)
run_cmd_with_logging(
f'cp {tmp_dir}/usr/mips-linux-gnu/lib/ld-2.23.so test/data/test_tmp_dir/lib/ld.so.1'
)
run_cmd_with_logging(
f'mv {tmp_dir}/usr/mips-linux-gnu/lib/libc-2.23.so test/data/test_tmp_dir_2/fact_extracted/lib/libc.so.6'
)
run_cmd_with_logging(
f'mv {tmp_dir}/usr/mips-linux-gnu/lib/ld-2.23.so test/data/test_tmp_dir_2/fact_extracted/lib/ld.so.1'
)
def main(skip_docker, radare, nginx, distribution):
# flask-security is not maintained anymore and replaced by flask-security-too.
# Since python package naming conflicts are not resolved automatically, we remove flask-security manually.
run_cmd_with_logging('sudo -EH pip3 uninstall -y flask-security')
install_pip_packages(PIP_DEPENDENCIES)
# installing web/js-frameworks
_install_css_and_js_files()
# create user database
_create_directory_for_authentication()
if nginx:
_install_nginx(distribution)
if not skip_docker:
_install_docker_images(radare)
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_frontend').unlink()
Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py')
return 0
def main(distribution):
xenial = distribution == 'xenial'
apt_install_packages('apt-transport-https')
logging.info('Updating system')
apt_update_sources()
apt_upgrade_system()
apt_autoremove_packages()
apt_clean_system()
# update submodules
git_output, git_code = execute_shell_command_get_return_code('(cd ../../ && git submodule foreach "git pull")')
if git_code != 0:
raise InstallationError('Failed to update submodules\n{}'.format(git_output))
# make bin dir
with suppress(FileExistsError):
os.mkdir('../bin')
# install python3 and general build stuff
apt_install_packages('python3', 'python3-dev', 'build-essential', 'automake', 'autoconf', 'libtool', 'git', 'unzip')
if not xenial:
pip3_install_packages('testresources')
# get a bugfree recent pip version
apt_remove_packages('python3-pip', 'python3-setuptools', 'python3-wheel')
apt_autoremove_packages()
install_pip('python3')
# install python2
apt_install_packages('python', 'python-dev')
apt_remove_packages('python-pip')
apt_autoremove_packages()
install_pip('python2')
# install general python dependencys
apt_install_packages('libmagic-dev')
apt_install_packages('libffi-dev', 'libfuzzy-dev')
pip3_install_packages('psutil')
pip3_install_packages('pytest==3.5.1', 'pytest-cov', 'pytest-pep8', 'pylint', 'python-magic', 'xmltodict', 'yara-python==3.7.0', 'appdirs')
pip3_install_packages('ssdeep')
pip3_install_packages('lief')
pip3_install_packages('requests')
# install python mongo bindings
pip3_install_packages('pymongo', 'pyyaml')
# VarietyJS (is executed by update_statistic.py)
try:
install_github_project('variety/variety', ['git checkout 2f4d815', 'mv -f variety.js ../../bin', 'mv -f spec ../../bin'])
except InstallationError as installation_error:
if 'Directory not empty' not in str(installation_error):
raise installation_error
logging.warning('variety spec not overwritten')
# installing common code modules
pip3_install_packages('hurry.filesize')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_files.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_mongo.git')
pip3_install_packages('git+https://github.com/mass-project/common_helper_encoder.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_filter.git')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to('src/start_fact.py')
return 0
def main(radare, nginx):
execute_shell_command_get_return_code(
'sudo -EH pip3 install werkzeug==0.14.1'
) # FIXME pinning werkzeug because of broken tests
pip3_install_packages('flask', 'flask_restful', 'flask_security',
'flask_sqlalchemy', 'flask-paginate', 'Flask-API',
'uwsgi', 'bcrypt', 'python-dateutil')
# installing web/js-frameworks
with OperateInDirectory('../web_interface/static'):
wget_static_web_content(
'https://github.com/twbs/bootstrap/releases/download/v3.3.7/bootstrap-3.3.7-dist.zip',
'.', [
'unzip -o bootstrap-3.3.7-dist.zip',
'rm bootstrap-3.3.7-dist.zip', 'rm -rf bootstrap',
'mv bootstrap-3.3.7-dist bootstrap'
], 'bootstrap')
_patch_bootstrap()
wget_static_web_content('http://code.jquery.com/jquery-1.12.0.min.js',
'bootstrap/js',
['mv jquery-1.12.0.min.js jquery.min.js'],
'jquery')
# wget_static_web_content('https://ajax.googleapis.com/ajax/libs/jquery/1.12.0/jquery.min.js', 'bootstrap/js', [], 'jquery')
wget_static_web_content(
'https://raw.githubusercontent.com/Eonasdan/bootstrap-datetimepicker/master/build/js/bootstrap-datetimepicker.min.js',
'bootstrap/js', [], 'datetimepicker js')
wget_static_web_content(
'https://raw.githubusercontent.com/Eonasdan/bootstrap-datetimepicker/master/build/css/bootstrap-datetimepicker.min.css',
'bootstrap/css', [], 'datetimepicker css')
wget_static_web_content(
'https://raw.githubusercontent.com/moment/moment/develop/moment.js',
'bootstrap/js', [], 'moment.js')
if not Path('bootstrap3-editable').exists():
wget_static_web_content(
'https://vitalets.github.io/x-editable/assets/zip/bootstrap3-editable-1.5.1.zip',
'.', [
'unzip -o bootstrap3-editable-1.5.1.zip',
'rm bootstrap3-editable-1.5.1.zip CHANGELOG.txt LICENSE-MIT README.md',
'rm -rf inputs-ext'
], 'x-editable')
if Path('jstree').is_dir():
shutil.rmtree('jstree')
wget_static_web_content(
'https://github.com/vakata/jstree/zipball/3.3.2', '.',
['unzip 3.3.2', 'rm 3.3.2', 'mv vakata* jstree'], 'jstree')
wget_static_web_content(
'https://code.angularjs.org/1.4.8/angular.min.js', '.', [],
'angularJS')
# wget_static_web_content('https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular.min.js', '.', [], 'angularJS')
wget_static_web_content(
'https://github.com/chartjs/Chart.js/releases/download/v2.3.0/Chart.js',
'.', [], 'charts.js')
_build_highlight_js()
# create user database
_create_directory_for_authentication()
if nginx:
apt_install_packages('nginx')
generate_and_install_certificate()
configure_nginx()
nginx_output, nginx_code = execute_shell_command_get_return_code(
'sudo nginx -s reload')
if nginx_code != 0:
raise InstallationError(
'Failed to start nginx\n{}'.format(nginx_output))
if radare:
logging.info('Initializing docker container for radare')
if check_if_command_in_path('docker-compose'):
with OperateInDirectory('radare'):
output, return_code = execute_shell_command_get_return_code(
'docker-compose build')
if return_code != 0:
raise InstallationError(
'Failed to initialize radare container:\n{}'.format(
output))
else:
raise InstallationError(
'docker-compose is not installed. Please (re-)run pre_install.sh'
)
# pull pdf report container
logging.info('Pulling pdf report container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_pdf_report')
if return_code != 0:
raise InstallationError(
'Failed to pull pdf report container:\n{}'.format(output))
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_frontend').unlink()
Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py')
return 0
def main(distribution): # pylint: disable=too-many-statements
if distribution == 'fedora':
logging.info('Updating system')
dnf_update_sources()
else:
apt_install_packages('apt-transport-https')
logging.info('Updating system')
apt_update_sources()
_, is_repository = execute_shell_command_get_return_code('git status')
if is_repository == 0:
# update submodules
git_output, git_code = execute_shell_command_get_return_code(
'(cd ../../ && git submodule foreach "git pull")')
if git_code != 0:
raise InstallationError(
'Failed to update submodules\n{}'.format(git_output))
else:
logging.warning(
'FACT is not set up using git. Note that *adding submodules* won\'t work!!'
)
# make bin dir
BIN_DIR.mkdir(exist_ok=True)
if distribution == 'fedora':
dnf_install_packages('python3')
dnf_install_packages('python3-devel')
# build-essential not available on fedora, getting equivalent
dnf_install_packages('gcc')
dnf_install_packages('gcc-c++')
dnf_install_packages('make')
dnf_install_packages('automake')
dnf_install_packages('kernel-devel')
dnf_install_packages('autoconf')
dnf_install_packages('libtool')
dnf_install_packages('git')
dnf_install_packages('unzip')
else:
# install python3 and general build stuff
apt_install_packages('python3', 'python3-dev', 'build-essential',
'automake', 'autoconf', 'libtool', 'git', 'unzip')
if not distribution == 'xenial':
pip3_install_packages('testresources')
if distribution == 'fedora':
dnf_remove_packages('python3-pip', 'python3-setuptools',
'python3-wheel')
else:
# get a bug free recent pip version
apt_remove_packages('python3-pip', 'python3-setuptools',
'python3-wheel')
install_pip('python3')
pip3_install_packages('setuptools==49.6.0')
if distribution != 'fedora':
# install python2
apt_install_packages('python', 'python-dev')
with suppress(InstallationError):
apt_remove_packages('python-pip')
install_pip('python2')
if distribution == 'fedora':
dnf_install_packages('file-devel')
dnf_install_packages('libffi-devel')
dnf_install_packages('python3-tlsh')
dnf_install_packages('python3-ssdeep')
else:
# install general python dependencies
apt_install_packages('libmagic-dev')
apt_install_packages('libfuzzy-dev')
apt_install_packages('python3-tlsh')
pip3_install_packages('ssdeep')
pip3_install_packages(
'git+https://github.com/fkie-cad/fact_helper_file.git')
pip3_install_packages('psutil')
pip3_install_packages('pytest==6.1.2', 'pytest-cov', 'pylint',
'python-magic', 'xmltodict', 'yara-python==3.7.0',
'appdirs')
pip3_install_packages(
'lief==0.10.1') # FIXME: unpin version when install bug is fixed
pip3_install_packages('requests')
# install python MongoDB bindings
pip3_install_packages('pymongo', 'pyyaml')
# VarietyJS (is executed by update_statistic.py)
if (BIN_DIR / 'spec').exists():
logging.warning('variety spec not overwritten')
else:
install_github_project('variety/variety', [
'git checkout 2f4d815', 'mv -f variety.js ../../bin',
'mv -f spec ../../bin'
])
# installing common code modules
pip3_install_packages('hurry.filesize')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_files.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_mongo.git')
pip3_install_packages(
'git+https://github.com/mass-project/common_helper_encoder.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_filter.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_process.git')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to(
'src/start_fact.py')
return 0
def main(radare, nginx):
pip3_install_packages('flask', 'flask_restful', 'flask_security',
'flask_sqlalchemy', 'flask-paginate', 'Flask-API',
'uwsgi', 'bcrypt', 'python-dateutil')
# installing web/js-frameworks
_install_and_patch_bootstrap()
with OperateInDirectory('../web_interface/static'):
if Path('jstree').is_dir():
shutil.rmtree('jstree')
wget_static_web_content(
'https://github.com/vakata/jstree/zipball/3.3.2', '.',
['unzip 3.3.2', 'rm 3.3.2', 'mv vakata* jstree'], 'jstree')
wget_static_web_content(
'https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular.min.js',
'.', [], 'angularJS')
wget_static_web_content(
'https://github.com/chartjs/Chart.js/releases/download/v2.3.0/Chart.js',
'.', [], 'charts.js')
_build_highlight_js()
# create user database
_create_directory_for_authentication()
if nginx:
_install_nginx()
if radare:
logging.info('Initializing docker container for radare')
execute_shell_command_get_return_code(
'virtualenv {}'.format(COMPOSE_VENV))
output, return_code = execute_shell_command_get_return_code(
'{} install -U docker-compose'.format(COMPOSE_VENV / 'bin' /
'pip'))
if return_code != 0:
raise InstallationError(
'Failed to set up virtualenv for docker-compose\n{}'.format(
output))
with OperateInDirectory('radare'):
output, return_code = execute_shell_command_get_return_code(
'{} build'.format(COMPOSE_VENV / 'bin' / 'docker-compose'))
if return_code != 0:
raise InstallationError(
'Failed to initialize radare container:\n{}'.format(
output))
# pull pdf report container
logging.info('Pulling pdf report container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_pdf_report')
if return_code != 0:
raise InstallationError(
'Failed to pull pdf report container:\n{}'.format(output))
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_frontend').unlink()
Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py')
return 0
else:
logging.info('done')
if __name__ == '__main__':
check_python_version()
args = _setup_argparser()
_setup_logging(args, debug_flag=args.debug)
welcome()
distribution = check_distribution()
none_chosen = not (args.frontend or args.db or args.backend)
installation_directory = str(
Path(Path(__file__).parent, 'install').absolute())
with OperateInDirectory(installation_directory):
common(distribution)
if args.frontend or none_chosen:
frontend(not args.no_radare, args.nginx)
if args.db or none_chosen:
db(distribution)
if args.backend or none_chosen:
backend()
if args.statistic_cronjob:
install_statistic_cronjob()
logging.info('installation complete')
logging.warning(
'If FACT does not start, reload the environment variables with: source /etc/profile'