def _rebuild_components(self, components): """ Cleans components to force them to be rebuilt. After successfully building an S2E component (e.g. QEMU, libs2e, Z3, etc.), the S2E Makefile will create a "stamp" in the S2E build directory. Subsequent builds will first check if a component's stamp exists, and if it does the build process will not rebuild. To force a rebuild, the stamp must be deleted. This function will delete the specified stamps to force a rebuild. """ # We are only interested in components that create a "stamp" in the # "stamps" directory. The "stamps" directory is stripped from the # component stamps = [ component[7:] for component in self._make('list').strip().split(' ') if component.startswith('stamps/') ] # The user can also specify "libs2e" rather than the complete # "libs2e-{release,debug}-make" stamp stamp_prefixes = {component.split('-')[0] for component in stamps} stamps_to_delete = [] for component in components: # Check if the specified component is valid "as is" if component in stamps: stamps_to_delete.append( self.env_path('build', 's2e', 'stamps', component)) continue # Check if the user has specified a valid component prefix # TODO: This will delete both the debug and release stamps (if they exist) if component in stamp_prefixes: stamps_to_delete.extend( glob.glob( self.env_path('build', 's2e', 'stamps', '%s-*' % component))) continue # If we've made it this far, the component is not valid raise CommandError('Component %s is not valid. Valid components ' 'are: %s' % (component, ', '.join(stamp_prefixes))) # Delete the stamps, ignoring any stamps that do not exist for stamp_to_delete in stamps_to_delete: try: os.remove(stamp_to_delete) logger.info('Deleted %s to force a rebuild', stamp_to_delete) except OSError: pass
def _get_tests(self): ts_dir = self.source_path('s2e', 'testsuite') if not os.path.isdir(ts_dir): raise CommandError( f'{ts_dir} does not exist. Please check that you updated the S2E source' ) tests = self._cmd_options['tests'] if not tests: tests = _get_tests(ts_dir) return tests
def _print_apps_list(self): img_build_dir = self.source_path(CONSTANTS['repos']['images']['build']) app_templates = get_app_templates(img_build_dir) if not app_templates: raise CommandError('No apps available to build. Make sure that ' '%s exists and is valid' % os.path.join(img_build_dir, 'apps.json')) print('Available applications:') for app_template, desc in sorted(app_templates.items()): for base_image in desc['base_images']: print(' * %s/%s - %s' % (base_image, app_template, desc['name']))
def _decompress_archive(self, archive_path): """ Decompress the given archive into the S2E environment's projects directory. """ try: logger.info('Decompressing archive %s', archive_path) tar(extract=True, xz=True, verbose=True, file=archive_path, directory=self.projects_path(), _fg=True, _out=sys.stdout, _err=sys.stderr) except ErrorReturnCode as e: raise CommandError('Failed to decompress project archive - %s' % e)
def _validate_and_create_project(self, options): self._target_path = options['target'] # Check that the analysis target is valid if not os.path.isfile(self._target_path): raise CommandError('Cannot analyze %s because it does not seem to ' 'exist' % self._target_path) # The default project name is the target program to be analyzed # (without any file extension) project_name = options['name'] if not project_name: project_name, _ = \ os.path.splitext(os.path.basename(self._target_path)) self._project_dir = self.env_path('projects', project_name) # Load the image JSON description. If it is not given, guess the image image = options['image'] img_build_dir = self.source_path(CONSTANTS['repos']['images']['build']) templates = get_image_templates(img_build_dir) if not image: image = self._guess_image(templates, options['target_arch']) self._img_json = self._get_or_download_image(templates, image, options['download_image']) # Check architecture consistency if not is_valid_arch(options['target_arch'], self._img_json['os']): raise CommandError('Binary is x86_64 while VM image is %s. Please ' 'choose another image' % self._img_json['os']['arch']) # Check if the project dir already exists # Do this after all checks have completed self._check_project_dir(options['force']) # Create the project directory os.mkdir(self._project_dir)
def handle(self, *args, **options): # Check the archive archive = options['archive'][0] if not os.path.isfile(archive): raise CommandError('%s is not a valid project archive' % archive) # Get the name of the project that we are importing project_name = _get_project_name(archive) logger.info('Importing project \'%s\' from %s', project_name, archive) # Check if a project with that name already exists project_path = self.projects_path(project_name) if os.path.isdir(project_path): if options['force']: logger.info('\'%s\' already exists - removing', project_name) shutil.rmtree(self.projects_path(project_name)) else: raise CommandError('\'%s\' already exists. Either remove this ' 'project or use the force option' % project_name) # Decompress the archive self._decompress_archive(archive) # Rewrite all of the exported files to fix their S2E environment paths logger.info('Rewriting project files') copy_and_rewrite_files(project_path, project_path, S2E_ENV_PLACEHOLDER, self.env_path()) with open(os.path.join(project_path, 'project.json'), 'r') as f: proj_desc = json.load(f) # Create a symlink to the guest tools directory self._symlink_guest_tools(project_path, proj_desc) # Create a symlink to guestfs (if it exists) if proj_desc.get('has_guestfs'): self._symlink_guestfs(project_path, proj_desc) logger.success('Project successfully imported from %s', archive)
def _call_post_project_gen_script(test_dir, test_config, project_dir): script = test_config.get('build-options', {}).get('post-project-generation-script', None) if not script: return script = os.path.join(test_dir, script) if not os.path.exists(script): raise CommandError('%s does not exist' % script) env = os.environ.copy() env['PROJECT_DIR'] = project_dir cmd = sh.Command(script).bake(_out=sys.stdout, _err=sys.stderr, _fg=True, _env=env) cmd()
def _gen_html(lcov_info_path, lcov_html_dir): """ Generate an LCOV HTML report. Returns the directory containing the HTML report. """ try: genhtml(lcov_info_path, output_directory=lcov_html_dir, _out=sys.stdout, _err=sys.stderr) except ErrorReturnCode as e: raise CommandError(e) from e
def _get_archive_rules(image_path, rule_names): if _has_app_image(rule_names): raise CommandError('Building archives of app images is not supported yet') archive_rules = [] for r in rule_names: archive_rules.append(os.path.join(image_path, f'{r}.tar.xz')) logger.info('The following archives will be built:') for a in archive_rules: logger.info(' * %s', a) return archive_rules
def handle(self, *args, **options): # Parse the ExecutionTracer.dat file(s) and generate an execution tree # for the given path IDs results_dir = self.project_path('s2e-last') execution_tree = parse_execution_tree(results_dir) if not execution_tree: raise CommandError('The execution trace is empty') syms = SymbolManager(self.install_path(), self.symbol_search_path) fp = ForkProfiler(execution_tree, syms) fp.get() fp.dump()
def _decompress(path): """ Decompress a .tar.xz file at the given path. The decompressed data will be located in the same directory as ``path``. """ logger.info('Decompressing %s', path) try: tar(extract=True, xz=True, verbose=True, file=path, directory=os.path.dirname(path), _fg=True, _out=sys.stdout, _err=sys.stderr) except ErrorReturnCode as e: raise CommandError(e)
def get_image_templates(img_build_dir): images = os.path.join(img_build_dir, 'images.json') try: with open(images, 'r') as f: template_json = json.load(f) except: raise CommandError('Could not parse %s. Something is wrong with the ' 'environment' % images) _validate_version(template_json, images) return template_json['images']
def _check_vmlinux(): """ Check that /boot/vmlinux* files are readable. This is important for guestfish. """ try: for f in glob.glob(os.path.join(os.sep, 'boot', 'vmlinu*')): with open(f, 'rb'): pass except IOError: raise CommandError('Make sure that the kernels in /boot are readable. ' 'This is required for guestfish. Please run the ' 'following command:\n\n' 'sudo chmod ugo+r /boot/vmlinu*') from None
def _get_templates(img_build_dir, filename, key): images = os.path.join(img_build_dir, filename) try: with open(images, 'r') as f: template_json = json.load(f) except: raise CommandError('Could not parse %s. Something is wrong with the ' 'environment' % images) from None _validate_version(template_json, images) return template_json[key]
def handle(self, *args, **options): # Exit if the makefile doesn't exist makefile = self.env_path('source', 'Makefile') if not os.path.isfile(makefile): raise CommandError('No makefile found in %s' % os.path.dirname(makefile)) # If the build directory doesn't exist, create it build_dir = self.env_path('build') if not os.path.isdir(build_dir): os.mkdir(build_dir) # Set up some environment variables env_vars = os.environ.copy() env_vars['S2E_PREFIX'] = self.install_path() components = options['components'] self._make = sh.Command('make').bake(directory=build_dir, file=makefile, _env=env_vars) # If the user has specified any components to rebuild, do this before # the build if components: self._rebuild_components(components) try: # Run make if options['debug']: logger.info('Building S2E (debug) in %s', build_dir) self._make('all-debug', _out=sys.stdout, _err=sys.stderr) else: logger.info('Building S2E (release) in %s', build_dir) self._make('install', _out=sys.stdout, _err=sys.stderr) except ErrorReturnCode as e: raise CommandError(e) from e logger.success('S2E built')
def _check_project_dir(project_dir, force=False): """ Check if a project directory with the given name already exists. If such a project exists, only continue if the ``force`` flag has been specified. """ if os.path.exists(project_dir) and not os.path.isdir(project_dir): raise CommandError( f'The path {project_dir} already exists and is a file.') if not os.path.isdir(project_dir): return if force: logger.info('\'%s\' already exists - removing', os.path.basename(project_dir)) shutil.rmtree(project_dir) else: raise CommandError('\'%s\' already exists. Either remove this ' 'project or use the force option' % os.path.basename(project_dir))
def _check_vmware(): """ Check if VMWare is running. VMware conflicts with S2E's requirement for KVM, so VMWare must *not* be running together with S2E. """ for proc in psutil.process_iter(): try: if proc.name() == 'vmware-vmx': raise CommandError('S2E uses KVM to build images. VMware ' 'is currently running, which is not ' 'compatible with KVM. Please close all ' 'VMware VMs and try again.') except NoSuchProcess: pass
def _initialize_disassembler(self): """ Initialize the Binary Ninja Python API. """ binaryninja_dir = self.config.get('binary_ninja', {}).get('dir') if not binaryninja_dir: raise CommandError('No path to Binary Ninja has been given in ' 's2e.yaml. Please add the following to your ' 's2e.yaml config to use this disassembler ' 'backend:\n\n' 'binary_ninja:\n' '\tdir: /path/to/binaryninja') binaryninja_py_dir = os.path.join(binaryninja_dir, 'python') if not os.path.isdir(binaryninja_py_dir): raise CommandError('Binary Ninja not found at %s' % binaryninja_dir) sys.path.append(binaryninja_py_dir) self._binaryninja_mod = importlib.import_module('binaryninja') self._bv = self._binaryninja_mod.BinaryViewType.get_view_of_file( self._project_desc['target_path'])
def _install_dependencies(interactive): """ Install S2E's dependencies. Only apt-get is supported for now. """ logger.info('Installing S2E dependencies') ubuntu_ver = _get_ubuntu_version() if not ubuntu_ver: return all_install_packages = CONSTANTS['dependencies']['common'] + \ CONSTANTS['dependencies'].get(f'ubuntu-{ubuntu_ver}', []) install_packages = [] deb_package_urls = [] for package in all_install_packages: if '.deb' in package: deb_package_urls.append(package) else: install_packages.append(package) install_opts = ['--no-install-recommends'] env = {} if not interactive: logger.info('Running install in non-interactive mode') env['DEBIAN_FRONTEND'] = 'noninteractive' install_opts = ['-y'] + install_opts try: # Enable 32-bit libraries dpkg_add_arch = sudo.bake('dpkg', add_architecture=True, _fg=True) dpkg_add_arch('i386') # Perform apt-get install apt_get = sudo.bake('apt-get', _fg=True, _env=env) apt_get.update() apt_get.install(install_opts + install_packages) except ErrorReturnCode as e: raise CommandError(e) # Install deb files at the end for url in deb_package_urls: logger.info('Installing deb %s...', url) filename, _ = urllib.request.urlretrieve(url) os.rename(filename, f'{filename}.deb') apt_get = sudo.bake('apt-get', _fg=True, _env=env) apt_get.install(install_opts + [f'{filename}.deb'])
def _decompress_archive(archive_path, dest_path): """ Decompress the given archive into the S2E environment's projects directory. """ try: with tempfile.TemporaryDirectory() as directory: logger.info('Decompressing archive %s to %s', archive_path, directory) tar(extract=True, xz=True, verbose=True, file=archive_path, directory=directory, _out=sys.stdout, _err=sys.stderr) old_path = os.path.join(directory, _get_project_name(archive_path)) shutil.move(old_path, dest_path) except ErrorReturnCode as e: raise CommandError('Failed to decompress project archive - %s' % e)
def _initialize_disassembler(self): """ Initialize Radare2 with r2pipe and perform the initial analysis. Sets the ``_r2`` attribute or raises an exception if Radare2/r2pipe cannot be found. """ try: import r2pipe except ImportError: raise CommandError('Unable to load r2pipe. Is Radare2/r2pipe ' 'installed?') self._r2 = r2pipe.open(self._project_desc['target_path']) self._r2.cmd('aaa')
def _gen_html(self, lcov_info_path): """ Generate an LCOV HTML report. Returns the directory containing the HTML report. """ from sh import genhtml, ErrorReturnCode lcov_html_dir = self.project_path('s2e-last', 'lcov') try: genhtml(lcov_info_path, output_directory=lcov_html_dir, _out=sys.stdout, _err=sys.stderr, _fg=True) except ErrorReturnCode as e: raise CommandError(e) return lcov_html_dir
def handle(self, *args, **options): command = options.pop('command', ()) if command == 'basic_block': # Select the disassembler backend disassembler = options.pop('disassembler', ()) if disassembler == 'ida': call_command(IDABasicBlockCoverage(), args, **options) elif disassembler == 'r2': call_command(R2BasicBlockCoverage(), args, **options) elif disassembler == 'binaryninja': call_command(BinaryNinjaBasicBlockCoverage(), args, **options) elif command == 'lcov': call_command(LineCoverage(), args, **options) else: raise CommandError('Invalid command %s' % command)
def _check_virtualbox(): """ Check if VirtualBox is running. VirtualBox conflicts with S2E's requirement for KVM, so VirtualBox must *not* be running together with S2E. """ # Adapted from https://github.com/giampaolo/psutil/issues/132#issuecomment-44017679 # to avoid race conditions for proc in psutil.process_iter(): try: if proc.name() == 'VBoxHeadless': raise CommandError('S2E uses KVM to build images. VirtualBox ' 'is currently running, which is not ' 'compatible with KVM. Please close all ' 'VirtualBox VMs and try again.') except NoSuchProcess: pass
def _create_archive(self, archive_path, export_dir): """ Create the final archive of all the exported project files. Args: archive_path: Path to the ``tar.xz`` archive. export_dir: Path to the directory containing the files to export. """ try: logger.info('Creating archive %s', archive_path) create_archive = tar.bake(create=True, xz=True, verbose=True, file=archive_path, directory=export_dir, _out=sys.stdout, _err=sys.stderr) create_archive(self._project_name) except ErrorReturnCode as e: raise CommandError('Failed to archive project - %s' % e) from e
def _get_disas_info(self, module, actual_module_path): # Check if a cached version of the disassembly information exists. # If it does, then we don't have to disassemble the binary (which # may take a long time for large binaries) disas_info = self._get_cached_disassembly_info(actual_module_path) # If no cached .disas file exists, generate a new one using the # given disassembler and cache the results if not disas_info: disas_info = self._get_disassembly_info(actual_module_path) if not disas_info: raise CommandError('No disassembly information found') # TODO: store the cached file along side the original file (e.g., in guestfs) self._save_disassembly_info(module, disas_info) return disas_info
def _call_post_project_gen_script(test_dir, test_config, options): script = test_config.get('build-options', {}).get('post-project-generation-script', None) if not script: return script = os.path.join(test_dir, script) if not os.path.exists(script): raise CommandError(f'{script} does not exist') env = os.environ.copy() env['PROJECT_DIR'] = options['project_path'] env['TARGET'] = options['target'].path env['TESTSUITE_ROOT'] = options['testsuite_root'] cmd = sh.Command(script).bake(_out=sys.stdout, _err=sys.stderr, _env=env) cmd()
def handle(self, *args, **options): target_path = self._project_desc['target_path'] target_name = self._project_desc['target'] # Get the translation block coverage information addr_counts = self._get_addr_coverage(target_name) if not addr_counts: raise CommandError('No translation block information found') file_line_info = _get_file_line_coverage(target_path, addr_counts) lcov_info_path = self._save_coverage_info(file_line_info) if options.get('html', False): lcov_html_dir = self._gen_html(lcov_info_path) return 'Line coverage saved to %s. An HTML report is available in %s' % (lcov_info_path, lcov_html_dir) return 'Line coverage saved to %s' % lcov_info_path
def _guess_image(self, templates, target_arch): """ At this stage, images may not exist, so we get the list of images from images.json (in the guest-images repo) rather than from the images folder. """ logger.info('No image was specified (-i option). Attempting to guess ' 'a suitable image for a %s binary', target_arch) for k, v in templates.iteritems(): if self._configurator.is_valid_binary(target_arch, self._target_path, v['os']): logger.warning('Found %s, which looks suitable for this ' 'binary. Please use -i if you want to use ' 'another image', k) return k raise CommandError('No suitable image available for this binary')
def _must_generate_test(self, test_name, test_config): build_options = test_config.get('build-options', {}) if build_options.get('windows-build-server', False): if not self._cmd_options.get('with_windows_build'): # Skip tests that require a windows build server if instructed logger.warn('Skipping test %s, because it requires a Windows build machine', test_name) return False host = self.config.get('windows_build_server', {}).get('host', '') user = self.config.get('windows_build_server', {}).get('user', '') if not host or not user: msg = 'Test %s requires a Windows build server.\n' \ 'Please check that your s2e.yaml file contains a valid Windows build server ' \ 'configuration. Refer to the following page for details on how to set up the server:\n' \ 'http://s2e.systems/docs/WindowsEnvSetup.html' % test_name raise CommandError(msg) return True