def install_files(self):
url_10_k_most_common = 'https://raw.githubusercontent.com/danielmiessler/SecLists/master/Passwords/Common-Credentials/10k-most-common.txt'
dest_10_k_most_common = f'{self.base_path}/internal/passwords/10k-most-common.txt'
urllib.request.urlretrieve(url_10_k_most_common, dest_10_k_most_common)
# FIXME This should be imported rather then executed
run_cmd_with_logging(
f'python3 {self.base_path}/internal/update_password_list.py')
def install_files(self):
# FIXME expose a function in setup_repository to to this directily
# from python instead of executing the script
os.chdir('internal')
if not os.access('cve_cpe.db', os.R_OK):
run_cmd_with_logging('python3 setup_repository.py')
run_cmd_with_logging('python3 setup_repository.py --update')
os.chdir(self.base_path)
def install_files(self):
with TemporaryDirectory(dir=str(self.base_path)) as tmp_dir:
# We use a mirror of an old database that should not change
if Path(f'{self.base_path}/bin/GeoLite2-City/').exists():
return
Path(f'{self.base_path}/bin').mkdir(exist_ok=True)
url_geolite = 'https://github.com/codeqq/geolite2-city-mirror/raw/master/GeoLite2-City.tar.gz'
dest_geolite = f'{tmp_dir}/GeoLite2-City.tar.gz'
urllib.request.urlretrieve(url_geolite, dest_geolite)
run_cmd_with_logging(f'tar -xf {dest_geolite} -C {tmp_dir}')
run_cmd_with_logging(
f'mv {tmp_dir}/GeoLite2-City_20191029 {self.base_path}/bin/GeoLite2-City'
)
def install_system_packages(self):
super().install_system_packages()
lshw_p = subprocess.run('lshw -c display'.split(),
stdout=PIPE,
stderr=PIPE,
check=True)
opencl_pkgs = []
if lshw_p.stdout == 'NVIDIA':
opencl_pkgs = ['nvidia-opencl-dev']
elif lshw_p.stdout == 'AMD':
opencl_pkgs = ['ocl-icd-opencl-dev', 'opencl-headers']
# Somehow we don't care about opencl on fedora
if self.distribution != 'fedora':
run_cmd_with_logging('sudo apt install -y ' +
' '.join(opencl_pkgs))
def main(distribution): # pylint: disable=too-many-statements
_update_package_sources(distribution)
_update_submodules()
BIN_DIR.mkdir(exist_ok=True)
apt_packages_path = INSTALL_DIR / 'apt-pkgs-common.txt'
dnf_packages_path = INSTALL_DIR / 'dnf-pkgs-common.txt'
if distribution != 'fedora':
pkgs = read_package_list_from_file(apt_packages_path)
apt_install_packages(*pkgs)
else:
pkgs = read_package_list_from_file(dnf_packages_path)
dnf_install_packages(*pkgs)
if not is_virtualenv():
install_pip()
elif distribution != 'fedora':
run_cmd_with_logging('pip install -U pip setuptools wheel')
else:
# on fedora, extra setuptools will break some system tools like selinux ones
run_cmd_with_logging('pip install -U pip wheel')
install_pip_packages(PIP_DEPENDENCIES)
# VarietyJS (is executed by update_statistic.py)
if (BIN_DIR / 'spec').exists():
logging.warning('variety spec not overwritten')
else:
install_github_project('variety/variety', [
'git checkout 2f4d815', 'mv -f variety.js ../../bin/',
'mv -f spec ../../bin/'
])
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_all_installed_fact_components').unlink()
Path('start_all_installed_fact_components').symlink_to(
'src/start_fact.py')
return 0
def main(skip_docker, radare, nginx, distribution):
# flask-security is not maintained anymore and replaced by flask-security-too.
# Since python package naming conflicts are not resolved automatically, we remove flask-security manually.
run_cmd_with_logging('sudo -EH pip3 uninstall -y flask-security')
install_pip_packages(PIP_DEPENDENCIES)
# installing web/js-frameworks
_install_css_and_js_files()
# create user database
_create_directory_for_authentication()
if nginx:
_install_nginx(distribution)
if not skip_docker:
_install_docker_images(radare)
with OperateInDirectory('../../'):
with suppress(FileNotFoundError):
Path('start_fact_frontend').unlink()
Path('start_fact_frontend').symlink_to('src/start_fact_frontend.py')
return 0
def build(self):
url_binwalk = f'https://github.com/ReFirmLabs/binwalk/archive/refs/tags/v{BINWALK_VERSION}.tar.gz'
dest_binwalk = f'binwalk-v{BINWALK_VERSION}.tar.gz'
urllib.request.urlretrieve(url_binwalk, dest_binwalk)
run_cmd_with_logging(f'tar -xf {dest_binwalk}')
with OperateInDirectory(f'binwalk-{BINWALK_VERSION}'):
if is_virtualenv():
run_cmd_with_logging('pip install -U .')
else:
run_cmd_with_logging('sudo -EH pip3 install -U .')
def build(self):
url_john = 'https://github.com/openwall/john/archive/1.9.0-Jumbo-1.tar.gz'
dest_john = '1.9.0-Jumbo-1.tar.gz'
urllib.request.urlretrieve(url_john, dest_john)
Path('john').mkdir(exist_ok=True)
run_cmd_with_logging(
f'tar -xf {dest_john} -C john --strip-components 1')
os.chdir('john/src')
run_cmd_with_logging('sudo ./configure -disable-openmp', shell=True)
run_cmd_with_logging('make -s clean && make -sj$(nproc)', shell=True)
os.chdir(self.build_path)
# Ensure the directory is empty
shutil.rmtree(f'{self.base_path}/bin', ignore_errors=True)
shutil.move('john/run', f'{self.base_path}/bin')
def install_files(self):
with TemporaryDirectory(dir=str(self.base_path)) as tmp_dir:
# We download a specific version of the package so no need to
# update downloaded files
if (Path(
f'{self.base_path}/test/data/test_tmp_dir/lib/libc.so.6'
).exists() and Path(
f'{self.base_path}/test/data/test_tmp_dir/lib/ld.so.1'
).exists() and Path(
f'{self.base_path}/test/data/test_tmp_dir_2/lib/libc.so.6'
).exists() and Path(
f'{self.base_path}/test/data/test_tmp_dir_2/lib/ld.so.1').
exists()):
return
url_libc6_mips = 'http://de.archive.ubuntu.com/ubuntu/pool/universe/c/cross-toolchain-base-ports/libc6-mips-cross_2.23-0ubuntu3cross1_all.deb'
dest_libc6_mips = f'{tmp_dir}/libc6-mips-cross_2.23-0ubuntu3cross1_all.deb'
urllib.request.urlretrieve(url_libc6_mips, dest_libc6_mips)
# We can't use `ar --output` because it was added in 2.34 but
# debian buster uses 2.31
with OperateInDirectory(tmp_dir):
run_cmd_with_logging(f'ar x {dest_libc6_mips} data.tar.xz')
run_cmd_with_logging(f'tar -xf {tmp_dir}/data.tar.xz -C {tmp_dir}')
Path('test/data/test_tmp_dir/lib').mkdir(exist_ok=True,
parents=True)
Path('test/data/test_tmp_dir_2/fact_extracted/lib').mkdir(
exist_ok=True, parents=True)
run_cmd_with_logging(
f'cp {tmp_dir}/usr/mips-linux-gnu/lib/libc-2.23.so test/data/test_tmp_dir/lib/libc.so.6'
)
run_cmd_with_logging(
f'cp {tmp_dir}/usr/mips-linux-gnu/lib/ld-2.23.so test/data/test_tmp_dir/lib/ld.so.1'
)
run_cmd_with_logging(
f'mv {tmp_dir}/usr/mips-linux-gnu/lib/libc-2.23.so test/data/test_tmp_dir_2/fact_extracted/lib/libc.so.6'
)
run_cmd_with_logging(
f'mv {tmp_dir}/usr/mips-linux-gnu/lib/ld-2.23.so test/data/test_tmp_dir_2/fact_extracted/lib/ld.so.1'
)
def install_docker_images(self):
run_cmd_with_logging(
f'docker build {self._get_build_args()} -t fact/qemu-exec:alpine-3.14 {self.base_path}/docker'
)
def install_docker_images(self):
run_cmd_with_logging(f'docker build -t fact/format_string_resolver {self.base_path}/docker')
def __init__(self):
super().__init__()
self.server = None
run_cmd_with_logging(f'docker-compose -f {COMPOSE_YAML} up -d')
def install_docker_images(self):
run_cmd_with_logging(f'docker build -t input-vectors {self.base_path}')
run_cmd_with_logging('docker pull fkiecad/radare-web-gui:latest')
def install_docker_images(self):
run_cmd_with_logging('docker pull fkiecad/cwe_checker:stable')
def shutdown(self):
super().shutdown()
if self.server:
self.server.shutdown()
run_cmd_with_logging(f'docker-compose -f {COMPOSE_YAML} down')
def do_last(self):
# We don't care about the return value here, e.g. on Ubunut clamav-freshclam.service already runs freshclam,
# this will cause another invocation of freshclam to fail.
# FIXME Are there other reasons?
run_cmd_with_logging('sudo -E freshclam', raise_error=False)
def install_docker_images(self):
run_cmd_with_logging('docker pull crazymax/linguist')
run_cmd_with_logging('docker pull cytopia/eslint')
def install_other_packages(self):
run_cmd_with_logging('sudo luarocks install argparse')
run_cmd_with_logging('sudo luarocks install luacheck')
run_cmd_with_logging('sudo luarocks install luafilesystem')
def install_docker_images(self):
run_cmd_with_logging(
f'docker build -t fs_metadata_mounting {self.base_path}/docker')
