def _rebuild_components(self, components):
"""
Cleans components to force them to be rebuilt.
After successfully building an S2E component (e.g. QEMU, libs2e, Z3,
etc.), the S2E Makefile will create a "stamp" in the S2E build
directory. Subsequent builds will first check if a component's stamp
exists, and if it does the build process will not rebuild. To force a
rebuild, the stamp must be deleted. This function will delete the
specified stamps to force a rebuild.
"""
# We are only interested in components that create a "stamp" in the
# "stamps" directory. The "stamps" directory is stripped from the
# component
stamps = [
component[7:]
for component in self._make('list').strip().split(' ')
if component.startswith('stamps/')
]
# The user can also specify "libs2e" rather than the complete
# "libs2e-{release,debug}-make" stamp
stamp_prefixes = {component.split('-')[0] for component in stamps}
stamps_to_delete = []
for component in components:
# Check if the specified component is valid "as is"
if component in stamps:
stamps_to_delete.append(
self.env_path('build', 's2e', 'stamps', component))
continue
# Check if the user has specified a valid component prefix
# TODO: This will delete both the debug and release stamps (if they exist)
if component in stamp_prefixes:
stamps_to_delete.extend(
glob.glob(
self.env_path('build', 's2e', 'stamps',
'%s-*' % component)))
continue
# If we've made it this far, the component is not valid
raise CommandError('Component %s is not valid. Valid components '
'are: %s' %
(component, ', '.join(stamp_prefixes)))
# Delete the stamps, ignoring any stamps that do not exist
for stamp_to_delete in stamps_to_delete:
try:
os.remove(stamp_to_delete)
logger.info('Deleted %s to force a rebuild', stamp_to_delete)
except OSError:
pass
def _get_tests(self):
ts_dir = self.source_path('s2e', 'testsuite')
if not os.path.isdir(ts_dir):
raise CommandError(
f'{ts_dir} does not exist. Please check that you updated the S2E source'
)
tests = self._cmd_options['tests']
if not tests:
tests = _get_tests(ts_dir)
return tests
def _print_apps_list(self):
img_build_dir = self.source_path(CONSTANTS['repos']['images']['build'])
app_templates = get_app_templates(img_build_dir)
if not app_templates:
raise CommandError('No apps available to build. Make sure that '
'%s exists and is valid' %
os.path.join(img_build_dir, 'apps.json'))
print('Available applications:')
for app_template, desc in sorted(app_templates.items()):
for base_image in desc['base_images']:
print(' * %s/%s - %s' % (base_image, app_template, desc['name']))
def _decompress_archive(self, archive_path):
"""
Decompress the given archive into the S2E environment's projects
directory.
"""
try:
logger.info('Decompressing archive %s', archive_path)
tar(extract=True, xz=True, verbose=True, file=archive_path,
directory=self.projects_path(), _fg=True, _out=sys.stdout,
_err=sys.stderr)
except ErrorReturnCode as e:
raise CommandError('Failed to decompress project archive - %s' % e)
def _validate_and_create_project(self, options):
self._target_path = options['target']
# Check that the analysis target is valid
if not os.path.isfile(self._target_path):
raise CommandError('Cannot analyze %s because it does not seem to '
'exist' % self._target_path)
# The default project name is the target program to be analyzed
# (without any file extension)
project_name = options['name']
if not project_name:
project_name, _ = \
os.path.splitext(os.path.basename(self._target_path))
self._project_dir = self.env_path('projects', project_name)
# Load the image JSON description. If it is not given, guess the image
image = options['image']
img_build_dir = self.source_path(CONSTANTS['repos']['images']['build'])
templates = get_image_templates(img_build_dir)
if not image:
image = self._guess_image(templates, options['target_arch'])
self._img_json = self._get_or_download_image(templates, image,
options['download_image'])
# Check architecture consistency
if not is_valid_arch(options['target_arch'], self._img_json['os']):
raise CommandError('Binary is x86_64 while VM image is %s. Please '
'choose another image' %
self._img_json['os']['arch'])
# Check if the project dir already exists
# Do this after all checks have completed
self._check_project_dir(options['force'])
# Create the project directory
os.mkdir(self._project_dir)
def handle(self, *args, **options):
# Check the archive
archive = options['archive'][0]
if not os.path.isfile(archive):
raise CommandError('%s is not a valid project archive' % archive)
# Get the name of the project that we are importing
project_name = _get_project_name(archive)
logger.info('Importing project \'%s\' from %s', project_name, archive)
# Check if a project with that name already exists
project_path = self.projects_path(project_name)
if os.path.isdir(project_path):
if options['force']:
logger.info('\'%s\' already exists - removing', project_name)
shutil.rmtree(self.projects_path(project_name))
else:
raise CommandError('\'%s\' already exists. Either remove this '
'project or use the force option' %
project_name)
# Decompress the archive
self._decompress_archive(archive)
# Rewrite all of the exported files to fix their S2E environment paths
logger.info('Rewriting project files')
copy_and_rewrite_files(project_path, project_path, S2E_ENV_PLACEHOLDER,
self.env_path())
with open(os.path.join(project_path, 'project.json'), 'r') as f:
proj_desc = json.load(f)
# Create a symlink to the guest tools directory
self._symlink_guest_tools(project_path, proj_desc)
# Create a symlink to guestfs (if it exists)
if proj_desc.get('has_guestfs'):
self._symlink_guestfs(project_path, proj_desc)
logger.success('Project successfully imported from %s', archive)
def _call_post_project_gen_script(test_dir, test_config, project_dir):
script = test_config.get('build-options', {}).get('post-project-generation-script', None)
if not script:
return
script = os.path.join(test_dir, script)
if not os.path.exists(script):
raise CommandError('%s does not exist' % script)
env = os.environ.copy()
env['PROJECT_DIR'] = project_dir
cmd = sh.Command(script).bake(_out=sys.stdout, _err=sys.stderr, _fg=True, _env=env)
cmd()
def _gen_html(lcov_info_path, lcov_html_dir):
"""
Generate an LCOV HTML report.
Returns the directory containing the HTML report.
"""
try:
genhtml(lcov_info_path,
output_directory=lcov_html_dir,
_out=sys.stdout,
_err=sys.stderr)
except ErrorReturnCode as e:
raise CommandError(e) from e
def _get_archive_rules(image_path, rule_names):
if _has_app_image(rule_names):
raise CommandError('Building archives of app images is not supported yet')
archive_rules = []
for r in rule_names:
archive_rules.append(os.path.join(image_path, f'{r}.tar.xz'))
logger.info('The following archives will be built:')
for a in archive_rules:
logger.info(' * %s', a)
return archive_rules
def handle(self, *args, **options):
# Parse the ExecutionTracer.dat file(s) and generate an execution tree
# for the given path IDs
results_dir = self.project_path('s2e-last')
execution_tree = parse_execution_tree(results_dir)
if not execution_tree:
raise CommandError('The execution trace is empty')
syms = SymbolManager(self.install_path(), self.symbol_search_path)
fp = ForkProfiler(execution_tree, syms)
fp.get()
fp.dump()
def _decompress(path):
"""
Decompress a .tar.xz file at the given path.
The decompressed data will be located in the same directory as ``path``.
"""
logger.info('Decompressing %s', path)
try:
tar(extract=True, xz=True, verbose=True, file=path,
directory=os.path.dirname(path), _fg=True, _out=sys.stdout,
_err=sys.stderr)
except ErrorReturnCode as e:
raise CommandError(e)
def get_image_templates(img_build_dir):
images = os.path.join(img_build_dir, 'images.json')
try:
with open(images, 'r') as f:
template_json = json.load(f)
except:
raise CommandError('Could not parse %s. Something is wrong with the '
'environment' % images)
_validate_version(template_json, images)
return template_json['images']
def _check_vmlinux():
"""
Check that /boot/vmlinux* files are readable. This is important for guestfish.
"""
try:
for f in glob.glob(os.path.join(os.sep, 'boot', 'vmlinu*')):
with open(f, 'rb'):
pass
except IOError:
raise CommandError('Make sure that the kernels in /boot are readable. '
'This is required for guestfish. Please run the '
'following command:\n\n'
'sudo chmod ugo+r /boot/vmlinu*') from None
def _get_templates(img_build_dir, filename, key):
images = os.path.join(img_build_dir, filename)
try:
with open(images, 'r') as f:
template_json = json.load(f)
except:
raise CommandError('Could not parse %s. Something is wrong with the '
'environment' % images) from None
_validate_version(template_json, images)
return template_json[key]
def handle(self, *args, **options):
# Exit if the makefile doesn't exist
makefile = self.env_path('source', 'Makefile')
if not os.path.isfile(makefile):
raise CommandError('No makefile found in %s' %
os.path.dirname(makefile))
# If the build directory doesn't exist, create it
build_dir = self.env_path('build')
if not os.path.isdir(build_dir):
os.mkdir(build_dir)
# Set up some environment variables
env_vars = os.environ.copy()
env_vars['S2E_PREFIX'] = self.install_path()
components = options['components']
self._make = sh.Command('make').bake(directory=build_dir,
file=makefile,
_env=env_vars)
# If the user has specified any components to rebuild, do this before
# the build
if components:
self._rebuild_components(components)
try:
# Run make
if options['debug']:
logger.info('Building S2E (debug) in %s', build_dir)
self._make('all-debug', _out=sys.stdout, _err=sys.stderr)
else:
logger.info('Building S2E (release) in %s', build_dir)
self._make('install', _out=sys.stdout, _err=sys.stderr)
except ErrorReturnCode as e:
raise CommandError(e) from e
logger.success('S2E built')
def _check_project_dir(project_dir, force=False):
"""
Check if a project directory with the given name already exists.
If such a project exists, only continue if the ``force`` flag has been
specified.
"""
if os.path.exists(project_dir) and not os.path.isdir(project_dir):
raise CommandError(
f'The path {project_dir} already exists and is a file.')
if not os.path.isdir(project_dir):
return
if force:
logger.info('\'%s\' already exists - removing',
os.path.basename(project_dir))
shutil.rmtree(project_dir)
else:
raise CommandError('\'%s\' already exists. Either remove this '
'project or use the force option' %
os.path.basename(project_dir))
def _check_vmware():
"""
Check if VMWare is running. VMware conflicts with S2E's requirement for KVM, so VMWare must
*not* be running together with S2E.
"""
for proc in psutil.process_iter():
try:
if proc.name() == 'vmware-vmx':
raise CommandError('S2E uses KVM to build images. VMware '
'is currently running, which is not '
'compatible with KVM. Please close all '
'VMware VMs and try again.')
except NoSuchProcess:
pass
def _initialize_disassembler(self):
"""
Initialize the Binary Ninja Python API.
"""
binaryninja_dir = self.config.get('binary_ninja', {}).get('dir')
if not binaryninja_dir:
raise CommandError('No path to Binary Ninja has been given in '
's2e.yaml. Please add the following to your '
's2e.yaml config to use this disassembler '
'backend:\n\n'
'binary_ninja:\n'
'\tdir: /path/to/binaryninja')
binaryninja_py_dir = os.path.join(binaryninja_dir, 'python')
if not os.path.isdir(binaryninja_py_dir):
raise CommandError('Binary Ninja not found at %s' %
binaryninja_dir)
sys.path.append(binaryninja_py_dir)
self._binaryninja_mod = importlib.import_module('binaryninja')
self._bv = self._binaryninja_mod.BinaryViewType.get_view_of_file(
self._project_desc['target_path'])
def _install_dependencies(interactive):
"""
Install S2E's dependencies.
Only apt-get is supported for now.
"""
logger.info('Installing S2E dependencies')
ubuntu_ver = _get_ubuntu_version()
if not ubuntu_ver:
return
all_install_packages = CONSTANTS['dependencies']['common'] + \
CONSTANTS['dependencies'].get(f'ubuntu-{ubuntu_ver}', [])
install_packages = []
deb_package_urls = []
for package in all_install_packages:
if '.deb' in package:
deb_package_urls.append(package)
else:
install_packages.append(package)
install_opts = ['--no-install-recommends']
env = {}
if not interactive:
logger.info('Running install in non-interactive mode')
env['DEBIAN_FRONTEND'] = 'noninteractive'
install_opts = ['-y'] + install_opts
try:
# Enable 32-bit libraries
dpkg_add_arch = sudo.bake('dpkg', add_architecture=True, _fg=True)
dpkg_add_arch('i386')
# Perform apt-get install
apt_get = sudo.bake('apt-get', _fg=True, _env=env)
apt_get.update()
apt_get.install(install_opts + install_packages)
except ErrorReturnCode as e:
raise CommandError(e)
# Install deb files at the end
for url in deb_package_urls:
logger.info('Installing deb %s...', url)
filename, _ = urllib.request.urlretrieve(url)
os.rename(filename, f'{filename}.deb')
apt_get = sudo.bake('apt-get', _fg=True, _env=env)
apt_get.install(install_opts + [f'{filename}.deb'])
def _decompress_archive(archive_path, dest_path):
"""
Decompress the given archive into the S2E environment's projects directory.
"""
try:
with tempfile.TemporaryDirectory() as directory:
logger.info('Decompressing archive %s to %s', archive_path, directory)
tar(extract=True, xz=True, verbose=True, file=archive_path,
directory=directory, _out=sys.stdout,
_err=sys.stderr)
old_path = os.path.join(directory, _get_project_name(archive_path))
shutil.move(old_path, dest_path)
except ErrorReturnCode as e:
raise CommandError('Failed to decompress project archive - %s' % e)
def _initialize_disassembler(self):
"""
Initialize Radare2 with r2pipe and perform the initial analysis.
Sets the ``_r2`` attribute or raises an exception if Radare2/r2pipe
cannot be found.
"""
try:
import r2pipe
except ImportError:
raise CommandError('Unable to load r2pipe. Is Radare2/r2pipe '
'installed?')
self._r2 = r2pipe.open(self._project_desc['target_path'])
self._r2.cmd('aaa')
def _gen_html(self, lcov_info_path):
"""
Generate an LCOV HTML report.
Returns the directory containing the HTML report.
"""
from sh import genhtml, ErrorReturnCode
lcov_html_dir = self.project_path('s2e-last', 'lcov')
try:
genhtml(lcov_info_path, output_directory=lcov_html_dir,
_out=sys.stdout, _err=sys.stderr, _fg=True)
except ErrorReturnCode as e:
raise CommandError(e)
return lcov_html_dir
def handle(self, *args, **options):
command = options.pop('command', ())
if command == 'basic_block':
# Select the disassembler backend
disassembler = options.pop('disassembler', ())
if disassembler == 'ida':
call_command(IDABasicBlockCoverage(), args, **options)
elif disassembler == 'r2':
call_command(R2BasicBlockCoverage(), args, **options)
elif disassembler == 'binaryninja':
call_command(BinaryNinjaBasicBlockCoverage(), args, **options)
elif command == 'lcov':
call_command(LineCoverage(), args, **options)
else:
raise CommandError('Invalid command %s' % command)
def _check_virtualbox():
"""
Check if VirtualBox is running. VirtualBox conflicts with S2E's requirement for KVM, so VirtualBox must
*not* be running together with S2E.
"""
# Adapted from https://github.com/giampaolo/psutil/issues/132#issuecomment-44017679
# to avoid race conditions
for proc in psutil.process_iter():
try:
if proc.name() == 'VBoxHeadless':
raise CommandError('S2E uses KVM to build images. VirtualBox '
'is currently running, which is not '
'compatible with KVM. Please close all '
'VirtualBox VMs and try again.')
except NoSuchProcess:
pass
def _create_archive(self, archive_path, export_dir):
"""
Create the final archive of all the exported project files.
Args:
archive_path: Path to the ``tar.xz`` archive.
export_dir: Path to the directory containing the files to export.
"""
try:
logger.info('Creating archive %s', archive_path)
create_archive = tar.bake(create=True, xz=True, verbose=True,
file=archive_path, directory=export_dir,
_out=sys.stdout,
_err=sys.stderr)
create_archive(self._project_name)
except ErrorReturnCode as e:
raise CommandError('Failed to archive project - %s' % e) from e
def _get_disas_info(self, module, actual_module_path):
# Check if a cached version of the disassembly information exists.
# If it does, then we don't have to disassemble the binary (which
# may take a long time for large binaries)
disas_info = self._get_cached_disassembly_info(actual_module_path)
# If no cached .disas file exists, generate a new one using the
# given disassembler and cache the results
if not disas_info:
disas_info = self._get_disassembly_info(actual_module_path)
if not disas_info:
raise CommandError('No disassembly information found')
# TODO: store the cached file along side the original file (e.g., in guestfs)
self._save_disassembly_info(module, disas_info)
return disas_info
def _call_post_project_gen_script(test_dir, test_config, options):
script = test_config.get('build-options',
{}).get('post-project-generation-script', None)
if not script:
return
script = os.path.join(test_dir, script)
if not os.path.exists(script):
raise CommandError(f'{script} does not exist')
env = os.environ.copy()
env['PROJECT_DIR'] = options['project_path']
env['TARGET'] = options['target'].path
env['TESTSUITE_ROOT'] = options['testsuite_root']
cmd = sh.Command(script).bake(_out=sys.stdout, _err=sys.stderr, _env=env)
cmd()
def handle(self, *args, **options):
target_path = self._project_desc['target_path']
target_name = self._project_desc['target']
# Get the translation block coverage information
addr_counts = self._get_addr_coverage(target_name)
if not addr_counts:
raise CommandError('No translation block information found')
file_line_info = _get_file_line_coverage(target_path, addr_counts)
lcov_info_path = self._save_coverage_info(file_line_info)
if options.get('html', False):
lcov_html_dir = self._gen_html(lcov_info_path)
return 'Line coverage saved to %s. An HTML report is available in %s' % (lcov_info_path, lcov_html_dir)
return 'Line coverage saved to %s' % lcov_info_path
def _guess_image(self, templates, target_arch):
"""
At this stage, images may not exist, so we get the list of images
from images.json (in the guest-images repo) rather than from the images
folder.
"""
logger.info('No image was specified (-i option). Attempting to guess '
'a suitable image for a %s binary', target_arch)
for k, v in templates.iteritems():
if self._configurator.is_valid_binary(target_arch, self._target_path, v['os']):
logger.warning('Found %s, which looks suitable for this '
'binary. Please use -i if you want to use '
'another image', k)
return k
raise CommandError('No suitable image available for this binary')
def _must_generate_test(self, test_name, test_config):
build_options = test_config.get('build-options', {})
if build_options.get('windows-build-server', False):
if not self._cmd_options.get('with_windows_build'):
# Skip tests that require a windows build server if instructed
logger.warn('Skipping test %s, because it requires a Windows build machine', test_name)
return False
host = self.config.get('windows_build_server', {}).get('host', '')
user = self.config.get('windows_build_server', {}).get('user', '')
if not host or not user:
msg = 'Test %s requires a Windows build server.\n' \
'Please check that your s2e.yaml file contains a valid Windows build server ' \
'configuration. Refer to the following page for details on how to set up the server:\n' \
'http://s2e.systems/docs/WindowsEnvSetup.html' % test_name
raise CommandError(msg)
return True
