def _install_yara():
logging.info('Installing yara')
# CAUTION: Yara python binding is installed in bootstrap_common, because it is needed in the frontend as well.
apt_install_packages('bison', 'flex', 'libmagic-dev')
if check_string_in_command('yara --version', '3.7.1'):
logging.info('skipping yara installation (already installed)')
else:
broken, output = False, ''
wget_output, wget_code = execute_shell_command_get_return_code('wget https://github.com/VirusTotal/yara/archive/v3.7.1.zip')
if wget_code != 0:
raise InstallationError('Error on yara download.\n{}'.format(wget_output))
zip_output, zip_code = execute_shell_command_get_return_code('unzip v3.7.1.zip')
if zip_code == 0:
yara_folder = [child for child in Path('.').iterdir() if 'yara-3.' in child.name][0]
with OperateInDirectory(yara_folder.name, remove=True):
os.chmod('bootstrap.sh', 0o775)
for command in ['./bootstrap.sh', './configure --enable-magic', 'make -j$(nproc)', 'sudo make install']:
output, return_code = execute_shell_command_get_return_code(command)
if return_code != 0:
broken = True
break
else:
raise InstallationError('Error on yara extraction.\n{}'.format(zip_output))
Path('v3.7.1.zip').unlink()
if broken:
raise InstallationError('Error in yara installation.\n{}'.format(output))
def main(distribution):
logging.info('Setting up mongo database')
if distribution == 'xenial':
_add_mongo_mirror_to_sources()
apt_update_sources()
apt_install_packages('mongodb-org')
else:
apt_install_packages('mongodb')
# creating DB directory
fact_db_directory = _get_db_directory()
mkdir_output, _ = execute_shell_command_get_return_code('sudo mkdir -p --mode=0744 {}'.format(fact_db_directory))
chown_output, chown_code = execute_shell_command_get_return_code('sudo chown {}:{} {}'.format(os.getuid(), os.getgid(), fact_db_directory))
if chown_code != 0:
raise InstallationError('Failed to set up database directory. Check if parent folder exists\n{}'.format('\n'.join((mkdir_output, chown_output))))
# initializing DB authentication
logging.info('Initialize database')
with OperateInDirectory('..'):
init_output, init_code = execute_shell_command_get_return_code('python3 init_database.py')
if init_code != 0:
raise InstallationError('Unable to initialize database\n{}'.format(init_output))
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_db').unlink()
Path('start_fact_db').symlink_to('src/start_fact_db.py')
return 0
def _update_package_sources(distribution):
logging.info('Updating system')
if distribution == 'fedora':
dnf_update_sources()
else:
apt_install_packages('apt-transport-https')
apt_update_sources()
def _install_yara(distribution): # pylint: disable=too-complex
logging.info('Installing yara')
# CAUTION: Yara python binding is installed in install/common.py, because it is needed in the frontend as well.
if distribution != 'fedora':
apt_install_packages('bison', 'flex')
if check_string_in_command_output('yara --version', '3.7.1'):
logging.info('skipping yara installation (already installed)')
return
wget_output, wget_code = execute_shell_command_get_return_code(
'wget https://github.com/VirusTotal/yara/archive/v3.7.1.zip')
if wget_code != 0:
raise InstallationError(f'Error on yara download.\n{wget_output}')
zip_output, return_code = execute_shell_command_get_return_code(
'unzip v3.7.1.zip')
Path('v3.7.1.zip').unlink()
if return_code != 0:
raise InstallationError(f'Error on yara extraction.\n{zip_output}')
yara_folder = [
child for child in Path('.').iterdir() if 'yara-3.' in child.name
][0]
with OperateInDirectory(yara_folder.name, remove=True):
os.chmod('bootstrap.sh', 0o775)
for command in [
'./bootstrap.sh', './configure --enable-magic',
'make -j$(nproc)', 'sudo make install'
]:
output, return_code = execute_shell_command_get_return_code(
command)
if return_code != 0:
raise InstallationError(
f'Error in yara installation.\n{output}')
def main(distribution):
xenial = distribution == 'xenial'
logging.info('Updating package lists')
apt_update_sources()
# Non python dependencies
apt_install_packages('build-essential', 'automake', 'autoconf', 'libtool')
# python dependencies
apt_install_packages('python3', 'python3-dev', 'python', 'python-dev', 'python-wheel', 'python-setuptools')
pip3_install_packages('pytest', 'pytest-cov', 'pytest-pep8')
if not xenial:
pip3_install_packages('testresources')
# make bin dir
with suppress(FileExistsError):
os.mkdir('../bin')
config = load_config('main.cfg')
data_folder = config.get('unpack', 'data_folder')
os.makedirs(str(Path(data_folder, 'files')), exist_ok=True)
os.makedirs(str(Path(data_folder, 'reports')), exist_ok=True)
return 0
def main(distribution):
# dependencies
apt_install_packages('libjpeg-dev', 'liblzma-dev', 'liblzo2-dev',
'zlib1g-dev', 'unzip', 'libffi-dev', 'libfuzzy-dev')
pip3_install_packages('pluginbase', 'entropy')
# removes due to compatibilty reasons
try:
apt_remove_packages('python-lzma')
pip2_remove_packages('pyliblzma')
except InstallationError:
logging.debug('python-lzma not removed because present already')
apt_install_packages('python-lzma')
# installing unpacker
_install_unpacker(distribution == 'xenial')
# installing common code modules
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_unpacking_classifier.git'
)
pip3_install_packages(
'git+https://github.com/fkie-cad/fact_helper_file.git')
# install plug-in dependencies
_install_plugins()
# configure environment
_edit_sudoers()
return 0
def _install_nginx():
apt_install_packages('nginx')
_generate_and_install_certificate()
_configure_nginx()
nginx_output, nginx_code = execute_shell_command_get_return_code('sudo nginx -s reload')
if nginx_code != 0:
raise InstallationError('Failed to start nginx\n{}'.format(nginx_output))
def install_dependencies(dependencies):
apt = dependencies.get('apt', [])
pip3 = dependencies.get('pip3', [])
github = dependencies.get('github', [])
apt_install_packages(*apt)
pip3_install_packages(*pip3)
for repo in github:
install_github_project(*repo)
def main(distribution):
# dependencies
if distribution == 'fedora':
dnf_install_packages('libjpeg-devel', 'openssl-devel',
'python3-tkinter')
else:
apt_install_packages('libjpeg-dev', 'libssl-dev', 'python3-tk')
pip3_install_packages('pluginbase', 'Pillow', 'cryptography', 'pyopenssl',
'matplotlib', 'docker', 'networkx')
# install yara
_install_yara(distribution)
# install checksec.sh
_install_checksec(distribution)
# build extraction docker container
logging.info('Building fact extraction container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_extractor')
if return_code != 0:
raise InstallationError(
f'Failed to pull extraction container:\n{output}')
# installing common code modules
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_yara.git')
pip3_install_packages(
'git+https://github.com/mass-project/common_analysis_base.git')
# install plug-in dependencies
_install_plugins(distribution)
# configure environment
_edit_environment()
# create directories
_create_firmware_directory()
# compiling yara signatures
compile_signatures()
_, yarac_return = execute_shell_command_get_return_code(
'yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc'
)
if yarac_return != 0:
raise InstallationError('Failed to compile yara test signatures')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_backend').unlink()
Path('start_fact_backend').symlink_to('src/start_fact_backend.py')
return 0
def main(distribution):
# dependencies
apt_install_packages('python-dev', 'python-setuptools')
apt_install_packages('libjpeg-dev', 'liblzma-dev', 'liblzo2-dev', 'zlib1g-dev')
apt_install_packages('libssl-dev python3-tk')
pip3_install_packages('pluginbase', 'Pillow', 'cryptography', 'pyopenssl', 'entropy', 'matplotlib')
apt_install_packages('python-pip')
# removes due to compatibilty reasons
apt_remove_packages('python-lzma')
pip2_remove_packages('pyliblzma')
apt_install_packages('python-lzma')
# install yara
_install_yara()
# installing unpacker
_install_unpacker(distribution == 'xenial')
# installing common code modules
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_process.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_yara.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_unpacking_classifier.git')
pip3_install_packages('git+https://github.com/mass-project/common_analysis_base.git')
# install plug-in dependencies
_install_plugins()
# compile custom magic file
with OperateInDirectory('../mime'):
cat_output, cat_code = execute_shell_command_get_return_code('cat custom_* > custommime')
file_output, file_code = execute_shell_command_get_return_code('file -C -m custommime')
mv_output, mv_code = execute_shell_command_get_return_code('mv -f custommime.mgc ../bin/')
if any(code != 0 for code in (cat_code, file_code, mv_code)):
raise InstallationError('Failed to properly compile magic file\n{}'.format('\n'.join((cat_output, file_output, mv_output))))
Path('custommime').unlink()
# configure environment
_edit_sudoers()
_edit_environment()
# create directories
_create_firmware_directory()
# compiling yara signatures
compile_signatures()
_, yarac_return = execute_shell_command_get_return_code('yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc')
if yarac_return != 0:
raise InstallationError('Failed to compile yara test signatures')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_backend').unlink()
Path('start_fact_backend').symlink_to('src/start_fact_backend.py')
return 0
def main(skip_docker, distribution):
apt_packages_path = INSTALL_DIR / 'apt-pkgs-backend.txt'
dnf_packages_path = INSTALL_DIR / 'dnf-pkgs-backend.txt'
if distribution != 'fedora':
pkgs = read_package_list_from_file(apt_packages_path)
apt_install_packages(*pkgs)
else:
pkgs = read_package_list_from_file(dnf_packages_path)
dnf_install_packages(*pkgs)
install_pip_packages(PIP_DEPENDENCIES)
# install yara
_install_yara()
_install_checksec()
if not skip_docker:
_install_docker_images()
# install plug-in dependencies
_install_plugins(distribution, skip_docker)
# configure environment
_edit_environment()
# create directories
_create_firmware_directory()
# compiling yara signatures
compile_signatures()
_, yarac_return = execute_shell_command_get_return_code('yarac -d test_flag=false ../test/unit/analysis/test.yara ../analysis/signatures/Yara_Base_Plugin.yc')
if yarac_return != 0:
raise InstallationError('Failed to compile yara test signatures')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_backend').unlink()
Path('start_fact_backend').symlink_to('src/start_fact_backend.py')
return 0
def _install_nginx(distribution):
if distribution != 'fedora':
apt_install_packages('nginx')
else:
dnf_install_packages('nginx')
_generate_and_install_certificate()
_configure_nginx()
if distribution == 'fedora':
execute_commands_and_raise_on_return_code([
'sudo restorecon -v /etc/nginx/fact.*',
'sudo semanage fcontext -at httpd_log_t "/var/log/fact(/.*)?" || true',
'sudo restorecon -v -R /var/log/fact'
],
error=
'restore selinux context')
nginx_output, nginx_code = execute_shell_command_get_return_code(
'sudo nginx -s reload')
if nginx_code != 0:
raise InstallationError(
'Failed to start nginx\n{}'.format(nginx_output))
def main(distribution): # pylint: disable=too-many-statements
_update_package_sources(distribution)
_update_submodules()
BIN_DIR.mkdir(exist_ok=True)
apt_packages_path = INSTALL_DIR / 'apt-pkgs-common.txt'
dnf_packages_path = INSTALL_DIR / 'dnf-pkgs-common.txt'
if distribution != 'fedora':
pkgs = read_package_list_from_file(apt_packages_path)
apt_install_packages(*pkgs)
else:
pkgs = read_package_list_from_file(dnf_packages_path)
dnf_install_packages(*pkgs)
if not is_virtualenv():
install_pip()
elif distribution != 'fedora':
run_cmd_with_logging('pip install -U pip setuptools wheel')
else:
# on fedora, extra setuptools will break some system tools like selinux ones
run_cmd_with_logging('pip install -U pip wheel')
install_pip_packages(PIP_DEPENDENCIES)
# VarietyJS (is executed by update_statistic.py)
if (BIN_DIR / 'spec').exists():
logging.warning('variety spec not overwritten')
else:
install_github_project('variety/variety', [
'git checkout 2f4d815', 'mv -f variety.js ../../bin/',
'mv -f spec ../../bin/'
])
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to(
'src/start_fact.py')
return 0
def _install_unpacker(xenial):
apt_install_packages('fakeroot')
# sasquatch unpacker
install_github_project('kartone/sasquatch', ['./build.sh'])
# ubi_reader
pip2_install_packages('python-lzo')
install_github_project('jrspruitt/ubi_reader',
['sudo python2 setup.py install --force'])
# binwalk
if xenial:
apt_install_packages('cramfsprogs')
apt_install_packages('libqt4-opengl', 'python3-opengl', 'python3-pyqt4',
'python3-pyqt4.qtopengl', 'mtd-utils', 'gzip',
'bzip2', 'tar', 'arj', 'lhasa', 'cabextract',
'cramfsswap', 'squashfs-tools', 'zlib1g-dev',
'liblzma-dev', 'liblzo2-dev', 'liblzo2-dev', 'xvfb')
apt_install_packages('libcapstone3', 'libcapstone-dev')
pip3_install_packages('pyqtgraph', 'capstone', 'cstruct', 'python-lzo',
'numpy', 'scipy')
install_github_project('sviehb/jefferson',
['sudo python3 setup.py install'])
_install_stuffit()
install_github_project('devttys0/binwalk',
['sudo python3 setup.py install --force'])
# patool and unpacking backends
pip2_install_packages('patool')
pip3_install_packages('patool')
apt_install_packages('openjdk-8-jdk')
if xenial:
apt_install_packages('zoo')
apt_install_packages('lrzip', 'cpio', 'unadf', 'rpm2cpio', 'lzop', 'lhasa',
'cabextract', 'zpaq', 'archmage', 'arj', 'xdms',
'rzip', 'lzip', 'unalz', 'unrar', 'unzip', 'gzip',
'nomarch', 'flac', 'unace', 'sharutils')
apt_install_packages('unar')
# firmware-mod-kit
install_github_project('rampageX/firmware-mod-kit', [
'git checkout 5e74fe9dd', '(cd src && sh configure && make)',
'cp src/yaffs2utils/unyaffs2 src/untrx src/tpl-tool/src/tpl-tool ../../bin/'
])
def install_dependencies(dependencies):
apt = dependencies.get('apt', [])
pip3 = dependencies.get('pip3', [])
apt_install_packages(*apt)
pip3_install_packages(*pip3)
def main(distribution): # pylint: disable=too-many-statements
if distribution == 'fedora':
logging.info('Updating system')
dnf_update_sources()
else:
apt_install_packages('apt-transport-https')
logging.info('Updating system')
apt_update_sources()
_, is_repository = execute_shell_command_get_return_code('git status')
if is_repository == 0:
# update submodules
git_output, git_code = execute_shell_command_get_return_code(
'(cd ../../ && git submodule foreach "git pull")')
if git_code != 0:
raise InstallationError(
'Failed to update submodules\n{}'.format(git_output))
else:
logging.warning(
'FACT is not set up using git. Note that *adding submodules* won\'t work!!'
)
# make bin dir
BIN_DIR.mkdir(exist_ok=True)
if distribution == 'fedora':
dnf_install_packages('python3')
dnf_install_packages('python3-devel')
# build-essential not available on fedora, getting equivalent
dnf_install_packages('gcc')
dnf_install_packages('gcc-c++')
dnf_install_packages('make')
dnf_install_packages('automake')
dnf_install_packages('kernel-devel')
dnf_install_packages('autoconf')
dnf_install_packages('libtool')
dnf_install_packages('git')
dnf_install_packages('unzip')
else:
# install python3 and general build stuff
apt_install_packages('python3', 'python3-dev', 'build-essential',
'automake', 'autoconf', 'libtool', 'git', 'unzip')
if not distribution == 'xenial':
pip3_install_packages('testresources')
if distribution == 'fedora':
dnf_remove_packages('python3-pip', 'python3-setuptools',
'python3-wheel')
else:
# get a bug free recent pip version
apt_remove_packages('python3-pip', 'python3-setuptools',
'python3-wheel')
install_pip('python3')
pip3_install_packages('setuptools==49.6.0')
if distribution != 'fedora':
# install python2
apt_install_packages('python', 'python-dev')
with suppress(InstallationError):
apt_remove_packages('python-pip')
install_pip('python2')
if distribution == 'fedora':
dnf_install_packages('file-devel')
dnf_install_packages('libffi-devel')
dnf_install_packages('python3-tlsh')
dnf_install_packages('python3-ssdeep')
else:
# install general python dependencies
apt_install_packages('libmagic-dev')
apt_install_packages('libfuzzy-dev')
apt_install_packages('python3-tlsh')
pip3_install_packages('ssdeep')
pip3_install_packages(
'git+https://github.com/fkie-cad/fact_helper_file.git')
pip3_install_packages('psutil')
pip3_install_packages('pytest==6.1.2', 'pytest-cov', 'pylint',
'python-magic', 'xmltodict', 'yara-python==3.7.0',
'appdirs')
pip3_install_packages(
'lief==0.10.1') # FIXME: unpin version when install bug is fixed
pip3_install_packages('requests')
# install python MongoDB bindings
pip3_install_packages('pymongo', 'pyyaml')
# VarietyJS (is executed by update_statistic.py)
if (BIN_DIR / 'spec').exists():
logging.warning('variety spec not overwritten')
else:
install_github_project('variety/variety', [
'git checkout 2f4d815', 'mv -f variety.js ../../bin',
'mv -f spec ../../bin'
])
# installing common code modules
pip3_install_packages('hurry.filesize')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_files.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_mongo.git')
pip3_install_packages(
'git+https://github.com/mass-project/common_helper_encoder.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_filter.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_process.git')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to(
'src/start_fact.py')
return 0
def main(distribution): # pylint: disable=too-many-statements
apt_install_packages('apt-transport-https')
logging.info('Updating system')
apt_update_sources()
apt_upgrade_system()
apt_autoremove_packages()
apt_clean_system()
# execute_shell_command_get_return_code 返回的值为 output, return_code两个值
# output 从 stdout 和 stderr 读取数据，直到文件结束符 返回一个 (stdout_data, stderr_data) 元组
# return_code None —— 子进程尚未结束； ==0 子进程正常退出； > 0 子进程异常退出，returncode对应于出错码； <0 子进程被信号杀掉了
_, is_repository = execute_shell_command_get_return_code('git status')
if is_repository == 0:
# update submodules
git_output, git_code = execute_shell_command_get_return_code(
'(cd ../../ && git submodule foreach "git pull")')
if git_code != 0:
raise InstallationError(
'Failed to update submodules\n{}'.format(git_output))
else:
logging.warning(
'FACT is not set up using git. Note that *adding submodules* won\'t work!!'
)
# make bin dir 在src目录下
with suppress(FileExistsError):
os.mkdir('../bin')
# install python3 and general build stuff
apt_install_packages('python3', 'python3-dev', 'build-essential',
'automake', 'autoconf', 'libtool', 'git', 'unzip')
if not distribution == 'xenial':
pip3_install_packages('testresources')
# get a bugfree recent pip version
apt_remove_packages('python3-pip', 'python3-setuptools', 'python3-wheel')
apt_autoremove_packages()
install_pip('python3')
# install python2
apt_install_packages('python', 'python-dev')
apt_remove_packages('python-pip')
apt_autoremove_packages()
install_pip('python2')
# install general python dependencys
apt_install_packages('libmagic-dev')
apt_install_packages('libfuzzy-dev')
apt_install_packages('python3-tlsh')
pip3_install_packages(
'git+https://github.com/fkie-cad/fact_helper_file.git')
pip3_install_packages('psutil')
pip3_install_packages('pytest==3.5.1', 'pytest-cov', 'pytest-pep8',
'pylint', 'python-magic', 'xmltodict',
'yara-python==3.7.0', 'appdirs')
pip3_install_packages('ssdeep')
pip3_install_packages('lief')
pip3_install_packages('requests')
# install python mongo bindings
pip3_install_packages('pymongo', 'pyyaml')
# VarietyJS (is executed by update_statistic.py)
try:
install_github_project('variety/variety', [
'git checkout 2f4d815', 'mv -f variety.js ../../bin',
'mv -f spec ../../bin'
])
except InstallationError as installation_error:
if 'Directory not empty' not in str(installation_error):
raise installation_error
logging.warning('variety spec not overwritten')
# installing common code modules
pip3_install_packages('hurry.filesize')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_files.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_mongo.git')
pip3_install_packages(
'git+https://github.com/mass-project/common_helper_encoder.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_filter.git')
pip3_install_packages(
'git+https://github.com/fkie-cad/common_helper_process.git')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to(
'src/start_fact.py')
return 0
def main(radare, nginx):
execute_shell_command_get_return_code(
'sudo -EH pip3 install werkzeug==0.14.1'
) # FIXME pinning werkzeug because of broken tests
pip3_install_packages('flask', 'flask_restful', 'flask_security',
'flask_sqlalchemy', 'flask-paginate', 'Flask-API',
'uwsgi', 'bcrypt', 'python-dateutil')
# installing web/js-frameworks
with OperateInDirectory('../web_interface/static'):
wget_static_web_content(
'https://github.com/twbs/bootstrap/releases/download/v3.3.7/bootstrap-3.3.7-dist.zip',
'.', [
'unzip -o bootstrap-3.3.7-dist.zip',
'rm bootstrap-3.3.7-dist.zip', 'rm -rf bootstrap',
'mv bootstrap-3.3.7-dist bootstrap'
], 'bootstrap')
_patch_bootstrap()
wget_static_web_content('http://code.jquery.com/jquery-1.12.0.min.js',
'bootstrap/js',
['mv jquery-1.12.0.min.js jquery.min.js'],
'jquery')
# wget_static_web_content('https://ajax.googleapis.com/ajax/libs/jquery/1.12.0/jquery.min.js', 'bootstrap/js', [], 'jquery')
wget_static_web_content(
'https://raw.githubusercontent.com/Eonasdan/bootstrap-datetimepicker/master/build/js/bootstrap-datetimepicker.min.js',
'bootstrap/js', [], 'datetimepicker js')
wget_static_web_content(
'https://raw.githubusercontent.com/Eonasdan/bootstrap-datetimepicker/master/build/css/bootstrap-datetimepicker.min.css',
'bootstrap/css', [], 'datetimepicker css')
wget_static_web_content(
'https://raw.githubusercontent.com/moment/moment/develop/moment.js',
'bootstrap/js', [], 'moment.js')
if not Path('bootstrap3-editable').exists():
wget_static_web_content(
'https://vitalets.github.io/x-editable/assets/zip/bootstrap3-editable-1.5.1.zip',
'.', [
'unzip -o bootstrap3-editable-1.5.1.zip',
'rm bootstrap3-editable-1.5.1.zip CHANGELOG.txt LICENSE-MIT README.md',
'rm -rf inputs-ext'
], 'x-editable')
if Path('jstree').is_dir():
shutil.rmtree('jstree')
wget_static_web_content(
'https://github.com/vakata/jstree/zipball/3.3.2', '.',
['unzip 3.3.2', 'rm 3.3.2', 'mv vakata* jstree'], 'jstree')
wget_static_web_content(
'https://code.angularjs.org/1.4.8/angular.min.js', '.', [],
'angularJS')
# wget_static_web_content('https://ajax.googleapis.com/ajax/libs/angularjs/1.4.8/angular.min.js', '.', [], 'angularJS')
wget_static_web_content(
'https://github.com/chartjs/Chart.js/releases/download/v2.3.0/Chart.js',
'.', [], 'charts.js')
_build_highlight_js()
# create user database
_create_directory_for_authentication()
if nginx:
apt_install_packages('nginx')
generate_and_install_certificate()
configure_nginx()
nginx_output, nginx_code = execute_shell_command_get_return_code(
'sudo nginx -s reload')
if nginx_code != 0:
raise InstallationError(
'Failed to start nginx\n{}'.format(nginx_output))
if radare:
logging.info('Initializing docker container for radare')
if check_if_command_in_path('docker-compose'):
with OperateInDirectory('radare'):
output, return_code = execute_shell_command_get_return_code(
'docker-compose build')
if return_code != 0:
raise InstallationError(
'Failed to initialize radare container:\n{}'.format(
output))
else:
raise InstallationError(
'docker-compose is not installed. Please (re-)run pre_install.sh'
)
# pull pdf report container
logging.info('Pulling pdf report container')
output, return_code = execute_shell_command_get_return_code(
'docker pull fkiecad/fact_pdf_report')
if return_code != 0:
raise InstallationError(
'Failed to pull pdf report container:\n{}'.format(output))
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_frontend').unlink()
Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py')
return 0
def _install_unpacker(xenial):
apt_install_packages('fakeroot')
# ---- sasquatch unpacker ----
# Original: devttys0/sasquatch
# Ubuntu 18.04 compatiblity issue in original source. Fixed in this fork:
install_github_project('kartone/sasquatch', ['./build.sh'])
# ubi_reader
pip2_install_packages('python-lzo')
install_github_project('jrspruitt/ubi_reader',
['sudo python2 setup.py install --force'])
# binwalk
if xenial:
# Replace by
# wget -O - https://sourceforge.net/projects/cramfs/files/cramfs/1.1/cramfs-1.1.tar.gz/download | tar -zxv
# cd cramfs-1.1
# sudo install cramfsck mkcramfs /usr/local/bin
# cd ..
# rm -rf cramfs-1.1
apt_install_packages('cramfsprogs')
apt_install_packages('libqt4-opengl', 'python3-opengl', 'python3-pyqt4',
'python3-pyqt4.qtopengl', 'mtd-utils', 'gzip',
'bzip2', 'tar', 'arj', 'lhasa', 'cabextract',
'cramfsswap', 'squashfs-tools', 'zlib1g-dev',
'liblzma-dev', 'liblzo2-dev', 'liblzo2-dev', 'xvfb')
apt_install_packages('libcapstone3', 'libcapstone-dev')
pip3_install_packages('pyqtgraph', 'capstone', 'cstruct', 'python-lzo',
'numpy', 'scipy')
install_github_project('sviehb/jefferson',
['sudo python3 setup.py install'])
_install_stuffit()
install_github_project('devttys0/binwalk',
['sudo python3 setup.py install --force'])
# patool and unpacking backends
pip2_install_packages('patool')
pip3_install_packages('patool')
apt_install_packages('openjdk-8-jdk')
if xenial:
apt_install_packages('zoo')
apt_install_packages('lrzip', 'cpio', 'unadf', 'rpm2cpio', 'lzop', 'lhasa',
'cabextract', 'zpaq', 'archmage', 'arj', 'xdms',
'rzip', 'lzip', 'unalz', 'unrar', 'unzip', 'gzip',
'nomarch', 'flac', 'unace', 'sharutils')
apt_install_packages('unar')
# firmware-mod-kit
install_github_project('rampageX/firmware-mod-kit', [
'(cd src && sh configure && make)',
'cp src/yaffs2utils/unyaffs2 src/untrx src/tpl-tool/src/tpl-tool ../../bin/'
])
def main(distribution):
xenial = distribution == 'xenial'
apt_install_packages('apt-transport-https')
logging.info('Updating system')
apt_update_sources()
apt_upgrade_system()
apt_autoremove_packages()
apt_clean_system()
# update submodules
git_output, git_code = execute_shell_command_get_return_code('(cd ../../ && git submodule foreach "git pull")')
if git_code != 0:
raise InstallationError('Failed to update submodules\n{}'.format(git_output))
# make bin dir
with suppress(FileExistsError):
os.mkdir('../bin')
# install python3 and general build stuff
apt_install_packages('python3', 'python3-dev', 'build-essential', 'automake', 'autoconf', 'libtool', 'git', 'unzip')
if not xenial:
pip3_install_packages('testresources')
# get a bugfree recent pip version
apt_remove_packages('python3-pip', 'python3-setuptools', 'python3-wheel')
apt_autoremove_packages()
install_pip('python3')
# install python2
apt_install_packages('python', 'python-dev')
apt_remove_packages('python-pip')
apt_autoremove_packages()
install_pip('python2')
# install general python dependencys
apt_install_packages('libmagic-dev')
apt_install_packages('libffi-dev', 'libfuzzy-dev')
pip3_install_packages('psutil')
pip3_install_packages('pytest==3.5.1', 'pytest-cov', 'pytest-pep8', 'pylint', 'python-magic', 'xmltodict', 'yara-python==3.7.0', 'appdirs')
pip3_install_packages('ssdeep')
pip3_install_packages('lief')
pip3_install_packages('requests')
# install python mongo bindings
pip3_install_packages('pymongo', 'pyyaml')
# VarietyJS (is executed by update_statistic.py)
try:
install_github_project('variety/variety', ['git checkout 2f4d815', 'mv -f variety.js ../../bin', 'mv -f spec ../../bin'])
except InstallationError as installation_error:
if 'Directory not empty' not in str(installation_error):
raise installation_error
logging.warning('variety spec not overwritten')
# installing common code modules
pip3_install_packages('hurry.filesize')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_files.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_mongo.git')
pip3_install_packages('git+https://github.com/mass-project/common_helper_encoder.git')
pip3_install_packages('git+https://github.com/fkie-cad/common_helper_filter.git')
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to('src/start_fact.py')
return 0
