def download_build_if_needed(dest, url): """Download and extract a build (if it's not already there).""" if os.path.exists(dest): return dest logger.info('Downloading build data...') gsutil_path = url.replace( 'https://storage.cloud.google.com/', 'gs://') common.gsutil('cp %s .' % gsutil_path, common.CLUSTERFUZZ_CACHE_DIR) filename = os.path.basename(gsutil_path) saved_file = os.path.join(common.CLUSTERFUZZ_CACHE_DIR, filename) tmp_dir_path = tempfile.mkdtemp(dir=common.CLUSTERFUZZ_TMP_DIR) common.execute('unzip', '-q %s -d %s' % (saved_file, tmp_dir_path), cwd='.') # args.gn is guaranteed to be in the wanted folder. In Chrome, it's under a # sub-directory. In Android, it's in the top dir. args_gn_path = common.find_file('args.gn', tmp_dir_path) shutil.copytree(os.path.dirname(args_gn_path), dest) logger.info('Cleaning up...') common.delete_if_exists(saved_file) common.delete_if_exists(tmp_dir_path)
def gn_gen(self): """Finalize args.gn and run `gn gen`.""" args_gn_path = os.path.join(self.get_build_dir_path(), ARGS_GN_FILENAME) common.ensure_dir(self.get_build_dir_path()) common.delete_if_exists(args_gn_path) # Let users edit the current args. content = serialize_gn_args(self.get_gn_args()) content = common.edit_if_needed( content, prefix='edit-args-gn-', comment='Edit %s before building.' % ARGS_GN_FILENAME, should_edit=self.options.edit_mode) # Write args to file and store. with open(args_gn_path, 'w') as f: f.write(content) logger.info( common.colorize('\nGenerating %s:\n%s\n', common.BASH_GREEN_MARKER), args_gn_path, content) common.execute( 'gn', 'gen %s' % (self.get_build_dir_path()), self.get_source_dir_path())
def install_deps(self): """Run download_gold_plugin.py.""" super(CfiChromiumBuilder, self).install_deps() if os.path.exists(os.path.join( self.source_directory, 'build/download_gold_plugin.py')): common.execute('build/download_gold_plugin.py', '', self.source_directory)
def reproduce_crash(binary_path, current_testcase): """Reproduces a crash by running the downloaded testcase against a binary.""" command = '%s %s %s' % (binary_path, current_testcase.reproduction_args, current_testcase.get_testcase_path()) common.execute(command, os.path.dirname(binary_path), environment=current_testcase.environment)
def test_with_ninja(self): """Ensure interpret_ninja_output is run when the ninja flag is set.""" x = mock.Mock() x.read.side_effect = ['part1', 'part2\n'] self.mock.Popen.return_value = mock.Mock(stdout=x, returncode=0) common.execute('ninja', 'do this plz', '~/working/directory', print_output=True, exit_on_error=True, env={ 'a': 'b', 1: 2, 'c': None }) self.assert_n_calls(1, [self.mock.interpret_ninja_output]) self.mock.Popen.assert_called_once_with('ninja do this plz', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd='~/working/directory', env={ 'a': 'b', 'OS': 'ENVIRON', '1': '2' }, preexec_fn=os.setsid)
def download_build(dest, url, binary_name): """Download and extract a build (if it's not already there).""" if os.path.exists(dest): return dest logger.info('Downloading build data...') common.ensure_dir(common.CLUSTERFUZZ_BUILDS_DIR) gsutil_path = url.replace( 'https://storage.cloud.google.com/', 'gs://') common.gsutil('cp %s .' % gsutil_path, common.CLUSTERFUZZ_CACHE_DIR) filename = os.path.basename(gsutil_path) saved_file = os.path.join(common.CLUSTERFUZZ_CACHE_DIR, filename) common.execute( 'unzip', '-q %s -d %s' % (saved_file, common.CLUSTERFUZZ_BUILDS_DIR), cwd=common.CLUSTERFUZZ_DIR) logger.info('Cleaning up...') os.remove(saved_file) os.rename(os.path.join( common.CLUSTERFUZZ_BUILDS_DIR, os.path.splitext(filename)[0]), dest) binary_location = os.path.join(dest, binary_name) stats = os.stat(binary_location) os.chmod(binary_location, stats.st_mode | stat.S_IEXEC)
def setup_gn_args(self): """Ensures that args.gn is set up properly.""" args_gn_location = os.path.join(self.build_directory, 'args.gn') if os.path.isfile(args_gn_location): os.remove(args_gn_location) if not os.path.exists(self.build_directory): os.makedirs(self.build_directory) lines = [] with open(os.path.join(self.build_dir_name(), 'args.gn'), 'r') as f: lines = [l.strip() for l in f.readlines()] with open(args_gn_location, 'w') as f: for line in lines: if 'goma_dir' in line: line = 'goma_dir = "%s"' % self.goma_dir f.write(line) f.write('\n') if self.gn_args_options: for k, v in self.gn_args_options.iteritems(): f.write('%s = %s\n' % (k, v)) common.execute('gn gen %s %s' % (self.gn_flags, self.build_directory), self.source_directory)
def download_build_data(self): """Downloads a build and saves it locally.""" build_dir = self.build_dir_name() binary_location = os.path.join(build_dir, self.binary_name) if os.path.exists(build_dir): return build_dir logger.info('Downloading build data...') if not os.path.exists(common.CLUSTERFUZZ_BUILDS_DIR): os.makedirs(common.CLUSTERFUZZ_BUILDS_DIR) gsutil_path = self.build_url.replace( 'https://storage.cloud.google.com/', 'gs://') common.execute( 'gsutil', 'cp %s .' % gsutil_path, common.CLUSTERFUZZ_CACHE_DIR) filename = os.path.split(gsutil_path)[1] saved_file = os.path.join(common.CLUSTERFUZZ_CACHE_DIR, filename) common.execute( 'unzip', '-q %s -d %s' % (saved_file, common.CLUSTERFUZZ_BUILDS_DIR), cwd=common.CLUSTERFUZZ_DIR) logger.info('Cleaning up...') os.remove(saved_file) os.rename(os.path.join(common.CLUSTERFUZZ_BUILDS_DIR, os.path.splitext(filename)[0]), build_dir) stats = os.stat(binary_location) os.chmod(binary_location, stats.st_mode | stat.S_IEXEC)
def download_build_data(self): """Downloads a build and saves it locally.""" build_dir = self.build_dir_name() if os.path.exists(build_dir): return build_dir print 'Downloading build data...' if not os.path.exists(CLUSTERFUZZ_BUILDS_DIR): os.makedirs(CLUSTERFUZZ_BUILDS_DIR) gsutil_path = self.build_url.replace( 'https://storage.cloud.google.com/', 'gs://') common.execute('gsutil cp %s .' % gsutil_path, CLUSTERFUZZ_DIR) filename = os.path.split(gsutil_path)[1] saved_file = os.path.join(CLUSTERFUZZ_DIR, filename) print 'Extracting...' zipped_file = zipfile.ZipFile(saved_file, 'r') zipped_file.extractall(CLUSTERFUZZ_BUILDS_DIR) zipped_file.close() print 'Cleaning up...' os.remove(saved_file) os.rename(os.path.join(CLUSTERFUZZ_BUILDS_DIR, os.path.splitext(filename)[0]), build_dir) binary_location = os.path.join(build_dir, self.target) stats = os.stat(binary_location) os.chmod(binary_location, stats.st_mode | stat.S_IEXEC)
def xdotool_command(self, command, display_name): """Run a command, returning its output.""" common.execute('xdotool', command, '.', env={'DISPLAY': display_name}, stdin=common.BlockStdin())
def get_testcase_path(self): """Downloads & returns the location of the testcase file.""" testcase_dir = self.testcase_dir_name() filename = os.path.join(testcase_dir, 'testcase%s' % self.file_extension) common.delete_if_exists(testcase_dir) os.makedirs(testcase_dir) logger.info('Downloading testcase data...') auth_header = common.get_stored_auth_header() # Do not use curl because curl doesn't support downloading an empty file. # See: https://github.com/google/clusterfuzz-tools/issues/326 args = ( '--no-verbose --waitretry=%s --retry-connrefused --content-disposition ' '--header="Authorization: %s" "%s"' % (DOWNLOAD_TIMEOUT, auth_header, CLUSTERFUZZ_TESTCASE_URL % self.id)) common.execute('wget', args, testcase_dir) downloaded_filename = os.listdir(testcase_dir)[0] filename = self.get_true_testcase_path(downloaded_filename) return filename
def ensure_goma(): """Ensures GOMA is installed and ready for use, and starts it.""" goma_dir = os.environ.get('GOMA_DIR', GOMA_DIR) if not os.path.isfile(os.path.join(goma_dir, 'goma_ctl.py')): raise error.GomaNotInstalledError() common.execute('python', 'goma_ctl.py ensure_start', goma_dir) return goma_dir
def install_deps(self): """Run download_gold_plugin.py.""" super(CfiMixin, self).install_deps() if os.path.exists(os.path.join( self.get_source_dir_path(), 'build/download_gold_plugin.py')): common.execute( 'build/download_gold_plugin.py', '', self.get_source_dir_path())
def install_build_deps_32bit(source_dir): """Run install-build-deps.sh.""" # preexec_fn is required to be None. Otherwise, it'd fail with: # 'sudo: no tty present and no askpass program specified'. common.execute( 'build/install-build-deps.sh', '--lib32 --syms --no-prompt', source_dir, stdout_transformer=output_transformer.Identity(), preexec_fn=None, redirect_stderr_to_stdout=True)
def install_deps(self): """Run all commands that only need to run once. This means the commands within this method are not required to be executed in a subsequential run.""" install_build_deps( self.get_source_dir_path(), include_lib32=self.include_lib32) common.execute('python', 'tools/clang/scripts/update.py', self.get_source_dir_path())
def gclient_sync(self): """Run gclient sync. This is separated from install_deps because it is needed in every build.""" common.execute( 'gclient', 'sync', self.get_source_dir_path(), # gclient sync sometimes asks a yes/no question (e.g. installing # Android SDK). stdin=common.StringStdin('y\ny\ny\n'), )
def reproduce_crash(binary_path, symbolizer_path, current_testcase, sanitizer): """Reproduces a crash by running the downloaded testcase against a binary.""" env = set_up_symbolizers_suppressions(current_testcase.environment, symbolizer_path, sanitizer) command = '%s %s %s' % (binary_path, current_testcase.reproduction_args, current_testcase.get_testcase_path()) common.execute(command, os.path.dirname(binary_path), environment=env)
def checkout_source_by_sha(self): """Checks out the correct revision.""" if self.get_current_sha() == self.git_sha: return command = 'git fetch && git checkout %s' % self.git_sha common.check_confirm('Proceed with the following command:\n%s in %s?' % (command, self.source_directory)) common.execute(command, self.source_directory)
def test_check_binary_fail(self): """Test check_binary fail.""" self.mock.check_binary.side_effect = error.NotInstalledError('cmd') with self.assertRaises(error.NotInstalledError) as cm: common.execute('cmd', 'aaa', '~/working/directory') self.assert_exact_calls(self.mock.check_binary, [mock.call('cmd', '~/working/directory')]) self.assertEqual(error.NotInstalledError.MESSAGE.format(binary='cmd'), cm.exception.message)
def gclient_runhooks_msan(source_dir, msan_track_origins): """Run gclient runhooks for msan.""" common.execute( 'gclient', 'runhooks', source_dir, env={ 'GYP_DEFINES': ( 'msan=1 msan_track_origins=%s ' 'use_prebuilt_instrumented_libraries=1' % (msan_track_origins or '2')) } )
def test_check_binary_fail(self): """Test check_binary fail.""" self.mock.check_binary.side_effect = common.NotInstalledError('cmd') with self.assertRaises(common.NotInstalledError) as cm: common.execute('cmd', 'aaa', '~/working/directory') self.assert_exact_calls(self.mock.check_binary, [mock.call('cmd', '~/working/directory')]) self.assertEqual( 'cmd is not found. Please install it or ensure the path is correct.', cm.exception.message)
def build_target(self): """Build the correct revision in the source directory.""" # Note: gclient sync must be run before setting up the gn args. common.execute('gclient', 'sync', self.source_directory) self.pre_build_steps() self.setup_gn_args() goma_cores = self.get_goma_cores() common.execute( 'ninja', "-w 'dupbuild=err' -C %s -j %i -l 15 %s" % ( self.build_directory, goma_cores, self.target), self.source_directory, capture_output=False)
def setup_gn_args(self): """Run the setup_gn_args and re-run hooks with special GYP_DEFINES.""" super(MsanV8Builder, self).setup_gn_args() args_hash = self.deserialize_gn_args(self.gn_args) msan_track_origins_value = (int(args_hash['msan_track_origins']) if 'msan_track_origins' in args_hash else 2) common.execute('gclient', 'runhooks', self.source_directory, env={'GYP_DEFINES': ('msan=1 msan_track_origins=%d ' 'use_prebuilt_instrumented_libraries=1') % msan_track_origins_value})
def checkout_source_by_sha(self): """Checks out the correct revision.""" _, current_sha = common.execute('git rev-parse HEAD', self.source_directory, print_output=False) if current_sha.strip() == self.git_sha: return command = 'git fetch && git checkout %s' % self.git_sha common.check_confirm('Proceed with the following command:\n%s in %s?' % (command, self.source_directory)) common.execute(command, self.source_directory)
def build_target(self): """Build the correct revision in the source directory.""" self.pre_build_steps() common.execute('gclient sync', self.source_directory) #Note: gclient sync must be run before setting up the gn args self.setup_gn_args() goma_cores = 10 * multiprocessing.cpu_count() common.execute( ("ninja -w 'dupbuild=err' -C %s -j %i -l %i %s" % (self.build_directory, goma_cores, goma_cores, self.target)), self.source_directory, capture_output=False)
def install_deps(self): """Install deps.""" super(ClankiumBuilder, self).install_deps() # preexec_fn is required to be None. Otherwise, it'd fail with: # 'sudo: no tty present and no askpass program specified'. # See why PATH is added: # https://github.com/google/clusterfuzz-tools/issues/497 common.execute( 'sudo', 'PATH=$PATH build/install-build-deps-android.sh', self.get_source_dir_path(), stdout_transformer=output_transformer.Identity(), preexec_fn=None, redirect_stderr_to_stdout=True)
def download_testcase(url): """Download the testcase into dest_dir.""" tmp_dir_path = tempfile.mkdtemp(dir=common.CLUSTERFUZZ_TMP_DIR) logger.info('Downloading testcase files...') auth_header = common.get_stored_auth_header() # Do not use curl because curl doesn't support downloading an empty file. # See: https://github.com/google/clusterfuzz-tools/issues/326 args = ( '--no-verbose --waitretry=%s --retry-connrefused --content-disposition ' '--header="Authorization: %s" "%s"' % (DOWNLOAD_TIMEOUT, auth_header, url)) common.execute('wget', args, tmp_dir_path) return os.path.join(tmp_dir_path, os.listdir(tmp_dir_path)[0])
def install_build_deps(source_dir, include_lib32): """Run install-build-deps.sh.""" flags = '--syms --no-prompt' if include_lib32: flags += ' --lib32' # preexec_fn is required to be None. Otherwise, it'd fail with: # 'sudo: no tty present and no askpass program specified'. # See why PATH is added: # https://github.com/google/clusterfuzz-tools/issues/497 common.execute( 'sudo', 'PATH=$PATH build/install-build-deps.sh %s' % flags, source_dir, stdout_transformer=output_transformer.Identity(), preexec_fn=None, redirect_stderr_to_stdout=True)
def build_target(self): """Build the correct revision in the source directory.""" if not self.options.disable_gclient: common.execute('gclient', 'sync', self.source_directory) self.pre_build_steps() self.setup_gn_args() goma_cores = self.get_goma_cores() common.execute('ninja', "-w 'dupbuild=err' -C %s -j %i -l 15 %s" % (self.build_directory, goma_cores, self.target), self.source_directory, capture_output=False, stdout_transformer=output_transformer.Ninja())
def find_windows_for_process(self, process_id, display_name): """Return visible windows belonging to a process.""" pids = self.get_process_ids(process_id) if not pids: return [] logger.info( 'Waiting for 30 seconds to ensure all windows appear: ' 'pid=%s, display=%s', pids, display_name) time.sleep(30) visible_windows = set() for pid in pids: _, windows = common.execute( 'xdotool', 'search --all --pid %s --onlyvisible --name ".*"' % pid, '.', env={'DISPLAY': display_name}, exit_on_error=False, print_command=False, print_output=False, stdin=common.BlockStdin()) for line in windows.splitlines(): if not line.isdigit(): continue visible_windows.add(line) logger.info('Found windows: %s', ', '.join(list(visible_windows))) return visible_windows