def download_build_if_needed(dest, url):
"""Download and extract a build (if it's not already there)."""
if os.path.exists(dest):
return dest
logger.info('Downloading build data...')
gsutil_path = url.replace(
'https://storage.cloud.google.com/', 'gs://')
common.gsutil('cp %s .' % gsutil_path, common.CLUSTERFUZZ_CACHE_DIR)
filename = os.path.basename(gsutil_path)
saved_file = os.path.join(common.CLUSTERFUZZ_CACHE_DIR, filename)
tmp_dir_path = tempfile.mkdtemp(dir=common.CLUSTERFUZZ_TMP_DIR)
common.execute('unzip', '-q %s -d %s' % (saved_file, tmp_dir_path), cwd='.')
# args.gn is guaranteed to be in the wanted folder. In Chrome, it's under a
# sub-directory. In Android, it's in the top dir.
args_gn_path = common.find_file('args.gn', tmp_dir_path)
shutil.copytree(os.path.dirname(args_gn_path), dest)
logger.info('Cleaning up...')
common.delete_if_exists(saved_file)
common.delete_if_exists(tmp_dir_path)
def gn_gen(self):
"""Finalize args.gn and run `gn gen`."""
args_gn_path = os.path.join(self.get_build_dir_path(), ARGS_GN_FILENAME)
common.ensure_dir(self.get_build_dir_path())
common.delete_if_exists(args_gn_path)
# Let users edit the current args.
content = serialize_gn_args(self.get_gn_args())
content = common.edit_if_needed(
content, prefix='edit-args-gn-',
comment='Edit %s before building.' % ARGS_GN_FILENAME,
should_edit=self.options.edit_mode)
# Write args to file and store.
with open(args_gn_path, 'w') as f:
f.write(content)
logger.info(
common.colorize('\nGenerating %s:\n%s\n', common.BASH_GREEN_MARKER),
args_gn_path, content)
common.execute(
'gn', 'gen %s' % (self.get_build_dir_path()),
self.get_source_dir_path())
def install_deps(self):
"""Run download_gold_plugin.py."""
super(CfiChromiumBuilder, self).install_deps()
if os.path.exists(os.path.join(
self.source_directory, 'build/download_gold_plugin.py')):
common.execute('build/download_gold_plugin.py', '', self.source_directory)
def reproduce_crash(binary_path, current_testcase):
"""Reproduces a crash by running the downloaded testcase against a binary."""
command = '%s %s %s' % (binary_path, current_testcase.reproduction_args,
current_testcase.get_testcase_path())
common.execute(command, os.path.dirname(binary_path),
environment=current_testcase.environment)
def test_with_ninja(self):
"""Ensure interpret_ninja_output is run when the ninja flag is set."""
x = mock.Mock()
x.read.side_effect = ['part1', 'part2\n']
self.mock.Popen.return_value = mock.Mock(stdout=x, returncode=0)
common.execute('ninja',
'do this plz',
'~/working/directory',
print_output=True,
exit_on_error=True,
env={
'a': 'b',
1: 2,
'c': None
})
self.assert_n_calls(1, [self.mock.interpret_ninja_output])
self.mock.Popen.assert_called_once_with('ninja do this plz',
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd='~/working/directory',
env={
'a': 'b',
'OS': 'ENVIRON',
'1': '2'
},
preexec_fn=os.setsid)
def download_build(dest, url, binary_name):
"""Download and extract a build (if it's not already there)."""
if os.path.exists(dest):
return dest
logger.info('Downloading build data...')
common.ensure_dir(common.CLUSTERFUZZ_BUILDS_DIR)
gsutil_path = url.replace(
'https://storage.cloud.google.com/', 'gs://')
common.gsutil('cp %s .' % gsutil_path, common.CLUSTERFUZZ_CACHE_DIR)
filename = os.path.basename(gsutil_path)
saved_file = os.path.join(common.CLUSTERFUZZ_CACHE_DIR, filename)
common.execute(
'unzip', '-q %s -d %s' % (saved_file, common.CLUSTERFUZZ_BUILDS_DIR),
cwd=common.CLUSTERFUZZ_DIR)
logger.info('Cleaning up...')
os.remove(saved_file)
os.rename(os.path.join(
common.CLUSTERFUZZ_BUILDS_DIR, os.path.splitext(filename)[0]), dest)
binary_location = os.path.join(dest, binary_name)
stats = os.stat(binary_location)
os.chmod(binary_location, stats.st_mode | stat.S_IEXEC)
def setup_gn_args(self):
"""Ensures that args.gn is set up properly."""
args_gn_location = os.path.join(self.build_directory, 'args.gn')
if os.path.isfile(args_gn_location):
os.remove(args_gn_location)
if not os.path.exists(self.build_directory):
os.makedirs(self.build_directory)
lines = []
with open(os.path.join(self.build_dir_name(), 'args.gn'), 'r') as f:
lines = [l.strip() for l in f.readlines()]
with open(args_gn_location, 'w') as f:
for line in lines:
if 'goma_dir' in line:
line = 'goma_dir = "%s"' % self.goma_dir
f.write(line)
f.write('\n')
if self.gn_args_options:
for k, v in self.gn_args_options.iteritems():
f.write('%s = %s\n' % (k, v))
common.execute('gn gen %s %s' % (self.gn_flags, self.build_directory),
self.source_directory)
def download_build_data(self):
"""Downloads a build and saves it locally."""
build_dir = self.build_dir_name()
binary_location = os.path.join(build_dir, self.binary_name)
if os.path.exists(build_dir):
return build_dir
logger.info('Downloading build data...')
if not os.path.exists(common.CLUSTERFUZZ_BUILDS_DIR):
os.makedirs(common.CLUSTERFUZZ_BUILDS_DIR)
gsutil_path = self.build_url.replace(
'https://storage.cloud.google.com/', 'gs://')
common.execute(
'gsutil', 'cp %s .' % gsutil_path, common.CLUSTERFUZZ_CACHE_DIR)
filename = os.path.split(gsutil_path)[1]
saved_file = os.path.join(common.CLUSTERFUZZ_CACHE_DIR, filename)
common.execute(
'unzip', '-q %s -d %s' % (saved_file, common.CLUSTERFUZZ_BUILDS_DIR),
cwd=common.CLUSTERFUZZ_DIR)
logger.info('Cleaning up...')
os.remove(saved_file)
os.rename(os.path.join(common.CLUSTERFUZZ_BUILDS_DIR,
os.path.splitext(filename)[0]), build_dir)
stats = os.stat(binary_location)
os.chmod(binary_location, stats.st_mode | stat.S_IEXEC)
def download_build_data(self):
"""Downloads a build and saves it locally."""
build_dir = self.build_dir_name()
if os.path.exists(build_dir):
return build_dir
print 'Downloading build data...'
if not os.path.exists(CLUSTERFUZZ_BUILDS_DIR):
os.makedirs(CLUSTERFUZZ_BUILDS_DIR)
gsutil_path = self.build_url.replace(
'https://storage.cloud.google.com/', 'gs://')
common.execute('gsutil cp %s .' % gsutil_path, CLUSTERFUZZ_DIR)
filename = os.path.split(gsutil_path)[1]
saved_file = os.path.join(CLUSTERFUZZ_DIR, filename)
print 'Extracting...'
zipped_file = zipfile.ZipFile(saved_file, 'r')
zipped_file.extractall(CLUSTERFUZZ_BUILDS_DIR)
zipped_file.close()
print 'Cleaning up...'
os.remove(saved_file)
os.rename(os.path.join(CLUSTERFUZZ_BUILDS_DIR,
os.path.splitext(filename)[0]), build_dir)
binary_location = os.path.join(build_dir, self.target)
stats = os.stat(binary_location)
os.chmod(binary_location, stats.st_mode | stat.S_IEXEC)
def xdotool_command(self, command, display_name):
"""Run a command, returning its output."""
common.execute('xdotool',
command,
'.',
env={'DISPLAY': display_name},
stdin=common.BlockStdin())
def get_testcase_path(self):
"""Downloads & returns the location of the testcase file."""
testcase_dir = self.testcase_dir_name()
filename = os.path.join(testcase_dir,
'testcase%s' % self.file_extension)
common.delete_if_exists(testcase_dir)
os.makedirs(testcase_dir)
logger.info('Downloading testcase data...')
auth_header = common.get_stored_auth_header()
# Do not use curl because curl doesn't support downloading an empty file.
# See: https://github.com/google/clusterfuzz-tools/issues/326
args = (
'--no-verbose --waitretry=%s --retry-connrefused --content-disposition '
'--header="Authorization: %s" "%s"' %
(DOWNLOAD_TIMEOUT, auth_header,
CLUSTERFUZZ_TESTCASE_URL % self.id))
common.execute('wget', args, testcase_dir)
downloaded_filename = os.listdir(testcase_dir)[0]
filename = self.get_true_testcase_path(downloaded_filename)
return filename
def ensure_goma():
"""Ensures GOMA is installed and ready for use, and starts it."""
goma_dir = os.environ.get('GOMA_DIR', GOMA_DIR)
if not os.path.isfile(os.path.join(goma_dir, 'goma_ctl.py')):
raise error.GomaNotInstalledError()
common.execute('python', 'goma_ctl.py ensure_start', goma_dir)
return goma_dir
def install_deps(self):
"""Run download_gold_plugin.py."""
super(CfiMixin, self).install_deps()
if os.path.exists(os.path.join(
self.get_source_dir_path(), 'build/download_gold_plugin.py')):
common.execute(
'build/download_gold_plugin.py', '', self.get_source_dir_path())
def install_build_deps_32bit(source_dir):
"""Run install-build-deps.sh."""
# preexec_fn is required to be None. Otherwise, it'd fail with:
# 'sudo: no tty present and no askpass program specified'.
common.execute(
'build/install-build-deps.sh', '--lib32 --syms --no-prompt',
source_dir, stdout_transformer=output_transformer.Identity(),
preexec_fn=None, redirect_stderr_to_stdout=True)
def install_deps(self):
"""Run all commands that only need to run once. This means the commands
within this method are not required to be executed in a subsequential
run."""
install_build_deps(
self.get_source_dir_path(), include_lib32=self.include_lib32)
common.execute('python', 'tools/clang/scripts/update.py',
self.get_source_dir_path())
def gclient_sync(self):
"""Run gclient sync. This is separated from install_deps because it is
needed in every build."""
common.execute(
'gclient', 'sync', self.get_source_dir_path(),
# gclient sync sometimes asks a yes/no question (e.g. installing
# Android SDK).
stdin=common.StringStdin('y\ny\ny\n'),
)
def reproduce_crash(binary_path, symbolizer_path, current_testcase, sanitizer):
"""Reproduces a crash by running the downloaded testcase against a binary."""
env = set_up_symbolizers_suppressions(current_testcase.environment,
symbolizer_path, sanitizer)
command = '%s %s %s' % (binary_path, current_testcase.reproduction_args,
current_testcase.get_testcase_path())
common.execute(command, os.path.dirname(binary_path), environment=env)
def checkout_source_by_sha(self):
"""Checks out the correct revision."""
if self.get_current_sha() == self.git_sha:
return
command = 'git fetch && git checkout %s' % self.git_sha
common.check_confirm('Proceed with the following command:\n%s in %s?' %
(command, self.source_directory))
common.execute(command, self.source_directory)
def test_check_binary_fail(self):
"""Test check_binary fail."""
self.mock.check_binary.side_effect = error.NotInstalledError('cmd')
with self.assertRaises(error.NotInstalledError) as cm:
common.execute('cmd', 'aaa', '~/working/directory')
self.assert_exact_calls(self.mock.check_binary,
[mock.call('cmd', '~/working/directory')])
self.assertEqual(error.NotInstalledError.MESSAGE.format(binary='cmd'),
cm.exception.message)
def gclient_runhooks_msan(source_dir, msan_track_origins):
"""Run gclient runhooks for msan."""
common.execute(
'gclient', 'runhooks', source_dir,
env={
'GYP_DEFINES': (
'msan=1 msan_track_origins=%s '
'use_prebuilt_instrumented_libraries=1'
% (msan_track_origins or '2'))
}
)
def test_check_binary_fail(self):
"""Test check_binary fail."""
self.mock.check_binary.side_effect = common.NotInstalledError('cmd')
with self.assertRaises(common.NotInstalledError) as cm:
common.execute('cmd', 'aaa', '~/working/directory')
self.assert_exact_calls(self.mock.check_binary,
[mock.call('cmd', '~/working/directory')])
self.assertEqual(
'cmd is not found. Please install it or ensure the path is correct.',
cm.exception.message)
def build_target(self):
"""Build the correct revision in the source directory."""
# Note: gclient sync must be run before setting up the gn args.
common.execute('gclient', 'sync', self.source_directory)
self.pre_build_steps()
self.setup_gn_args()
goma_cores = self.get_goma_cores()
common.execute(
'ninja',
"-w 'dupbuild=err' -C %s -j %i -l 15 %s" % (
self.build_directory, goma_cores, self.target),
self.source_directory, capture_output=False)
def setup_gn_args(self):
"""Run the setup_gn_args and re-run hooks with special GYP_DEFINES."""
super(MsanV8Builder, self).setup_gn_args()
args_hash = self.deserialize_gn_args(self.gn_args)
msan_track_origins_value = (int(args_hash['msan_track_origins'])
if 'msan_track_origins' in args_hash
else 2)
common.execute('gclient', 'runhooks', self.source_directory,
env={'GYP_DEFINES':
('msan=1 msan_track_origins=%d '
'use_prebuilt_instrumented_libraries=1') %
msan_track_origins_value})
def checkout_source_by_sha(self):
"""Checks out the correct revision."""
_, current_sha = common.execute('git rev-parse HEAD',
self.source_directory,
print_output=False)
if current_sha.strip() == self.git_sha:
return
command = 'git fetch && git checkout %s' % self.git_sha
common.check_confirm('Proceed with the following command:\n%s in %s?' %
(command, self.source_directory))
common.execute(command, self.source_directory)
def build_target(self):
"""Build the correct revision in the source directory."""
self.pre_build_steps()
common.execute('gclient sync', self.source_directory)
#Note: gclient sync must be run before setting up the gn args
self.setup_gn_args()
goma_cores = 10 * multiprocessing.cpu_count()
common.execute(
("ninja -w 'dupbuild=err' -C %s -j %i -l %i %s" %
(self.build_directory, goma_cores, goma_cores, self.target)),
self.source_directory,
capture_output=False)
def install_deps(self):
"""Install deps."""
super(ClankiumBuilder, self).install_deps()
# preexec_fn is required to be None. Otherwise, it'd fail with:
# 'sudo: no tty present and no askpass program specified'.
# See why PATH is added:
# https://github.com/google/clusterfuzz-tools/issues/497
common.execute(
'sudo', 'PATH=$PATH build/install-build-deps-android.sh',
self.get_source_dir_path(),
stdout_transformer=output_transformer.Identity(),
preexec_fn=None, redirect_stderr_to_stdout=True)
def download_testcase(url):
"""Download the testcase into dest_dir."""
tmp_dir_path = tempfile.mkdtemp(dir=common.CLUSTERFUZZ_TMP_DIR)
logger.info('Downloading testcase files...')
auth_header = common.get_stored_auth_header()
# Do not use curl because curl doesn't support downloading an empty file.
# See: https://github.com/google/clusterfuzz-tools/issues/326
args = (
'--no-verbose --waitretry=%s --retry-connrefused --content-disposition '
'--header="Authorization: %s" "%s"' %
(DOWNLOAD_TIMEOUT, auth_header, url))
common.execute('wget', args, tmp_dir_path)
return os.path.join(tmp_dir_path, os.listdir(tmp_dir_path)[0])
def install_build_deps(source_dir, include_lib32):
"""Run install-build-deps.sh."""
flags = '--syms --no-prompt'
if include_lib32:
flags += ' --lib32'
# preexec_fn is required to be None. Otherwise, it'd fail with:
# 'sudo: no tty present and no askpass program specified'.
# See why PATH is added:
# https://github.com/google/clusterfuzz-tools/issues/497
common.execute(
'sudo', 'PATH=$PATH build/install-build-deps.sh %s' % flags, source_dir,
stdout_transformer=output_transformer.Identity(),
preexec_fn=None, redirect_stderr_to_stdout=True)
def build_target(self):
"""Build the correct revision in the source directory."""
if not self.options.disable_gclient:
common.execute('gclient', 'sync', self.source_directory)
self.pre_build_steps()
self.setup_gn_args()
goma_cores = self.get_goma_cores()
common.execute('ninja',
"-w 'dupbuild=err' -C %s -j %i -l 15 %s" %
(self.build_directory, goma_cores, self.target),
self.source_directory,
capture_output=False,
stdout_transformer=output_transformer.Ninja())
def find_windows_for_process(self, process_id, display_name):
"""Return visible windows belonging to a process."""
pids = self.get_process_ids(process_id)
if not pids:
return []
logger.info(
'Waiting for 30 seconds to ensure all windows appear: '
'pid=%s, display=%s', pids, display_name)
time.sleep(30)
visible_windows = set()
for pid in pids:
_, windows = common.execute(
'xdotool',
'search --all --pid %s --onlyvisible --name ".*"' % pid,
'.',
env={'DISPLAY': display_name},
exit_on_error=False,
print_command=False,
print_output=False,
stdin=common.BlockStdin())
for line in windows.splitlines():
if not line.isdigit():
continue
visible_windows.add(line)
logger.info('Found windows: %s', ', '.join(list(visible_windows)))
return visible_windows