def reset_and_run_testcase(testcase_id, category, release):
"""Resets the chromium repo and runs the testcase."""
delete_if_exists(CHROMIUM_OUT)
delete_if_exists(CLUSTERFUZZ_CACHE_DIR)
process.call('git checkout -f HEAD', cwd=CHROMIUM_SRC)
# Clean untracked files. Because untracked files in submodules are not removed
# with `git checkout -f HEAD`. `git clean -ffdd` also cleans uncommitted
# changes (but not ignored files). Anecdotally, ignored files (which we don't
# handle) cause failure in `gclient sync` and `gn gen`. But we cannot do
# `git clean -ffddx` because we would hit the git rate-limit.
process.call('git clean -ffdd', cwd=CHROMIUM_SRC)
clean_third_party()
version = prepare_binary_and_get_version(release)
for opts in TEST_OPTIONS:
update_auth_header()
return_code = run_testcase(testcase_id, opts=opts)
logs = read_logs()
stackdriver_logging.send_run(
testcase_id, category, version, release, return_code, logs, opts)
def test_error(self):
"""Test raising exception if returncode is not zero."""
self.popen.returncode = 1
self.popen.communicate.return_value = ('Test', None)
with self.assertRaises(subprocess.CalledProcessError) as cm:
process.call('test', cwd='path', env={'NEW': '2'}, capture=False)
self.mock.Popen.assert_called_once_with('test',
shell=True,
cwd='path',
env={
'TEST': '1',
'NEW': '2'
},
stdout=None,
preexec_fn=os.setsid)
self.popen.communicate.assert_called_once_with()
self.mock.store_last_pid.assert_called_once_with(123)
self.assert_exact_calls(self.mock.kill_last_pid, [mock.call()] * 2)
self.assertEqual(1, cm.exception.returncode)
self.assertEqual('Test', cm.exception.output)
self.assertEqual('test', cm.exception.cmd)
self.mock.Thread.assert_called_once_with(
target=process.kill_when_timeout,
args=(self.popen, process.DEFAULT_TIMEOUT))
self.mock.Thread.return_value.start.assert_called_once_with()
def clean_third_party():
"""Clean third_party/ dir. The children of third_party/ are git-ignored.
Therefore, git-clean doesn't work. We can't do `git clean -x` either
because we would need to pull everything again and hit git's rate limit.
Therefore, we go into each child (which is a git repo) and run
clean the repo manually."""
for root, dirs, _ in os.walk(os.path.join(CHROMIUM_SRC, 'third_party')):
for name in dirs:
basename = os.path.basename(name)
if basename != '.git':
continue
parent = os.path.dirname(name)
process.call('git checkout HEAD -f', cwd=os.path.join(root, parent))
process.call('git clean -ffdd', cwd=os.path.join(root, parent))
def test_not_capture(self):
"""Test not capture."""
self.popen.returncode = 0
self.popen.communicate.return_value = (None, None)
self.assertEqual((0, None),
process.call('test',
cwd='path',
env={'NEW': '2'},
capture=False))
self.mock.Popen.assert_called_once_with('test',
shell=True,
cwd='path',
env={
'TEST': '1',
'NEW': '2'
},
stdout=None,
preexec_fn=os.setsid)
self.popen.communicate.assert_called_once_with()
self.mock.store_last_pid.assert_called_once_with(123)
self.assert_exact_calls(self.mock.kill_last_pid, [mock.call()] * 2)
self.mock.Thread.assert_called_once_with(
target=process.kill_when_timeout,
args=(self.popen, process.DEFAULT_TIMEOUT))
self.mock.Thread.return_value.start.assert_called_once_with()
def build_master_and_get_version():
"""Checks out the latest master build and creates a new binary."""
if not os.path.exists(TOOL_SOURCE):
process.call(
'git clone https://github.com/google/clusterfuzz-tools.git', cwd=HOME)
process.call('git fetch', cwd=TOOL_SOURCE)
process.call('git checkout origin/master -f', cwd=TOOL_SOURCE)
process.call('./pants binary tool:clusterfuzz-ci', cwd=TOOL_SOURCE,
env={'HOME': HOME})
delete_if_exists(BINARY_LOCATION)
shutil.copy(os.path.join(TOOL_SOURCE, 'dist', 'clusterfuzz-ci.pex'),
BINARY_LOCATION)
# The full SHA is too long and unpleasant to show in logs. So, we use the
# first 7 characters of the SHA instead.
return process.call(
'git rev-parse HEAD', capture=True, cwd=TOOL_SOURCE)[1].strip()[:7]
def test_capture(self):
"""Test capturing output."""
self.popen.returncode = 0
self.popen.communicate.return_value = ('Test', None)
self.assertEqual(
(0, 'Test'),
process.call('test', cwd='path', env={'NEW': '2'}, capture=True))
self.mock.Popen.assert_called_once_with(
'test', shell=True, cwd='path', env={'TEST': '1', 'NEW': '2'},
stdout=subprocess.PIPE, preexec_fn=os.setsid)
self.popen.communicate.assert_called_once_with()
self.mock.store_last_pid.assert_called_once_with(123)
self.assert_exact_calls(self.mock.kill_last_pid, [mock.call()] * 2)
def run_testcase(testcase_id, opts):
"""Attempts to reproduce a testcase."""
return_code, _ = process.call(
('%s reproduce %s %s' % (BINARY_LOCATION, testcase_id, opts)).strip(),
cwd=HOME,
env={
'CF_QUIET': '1',
'CHROMIUM_SRC': CHROMIUM_SRC,
'GOMA_GCE_SERVICE_ACCOUNT': 'default',
'PATH': '%s:%s' % (os.environ['PATH'], DEPOT_TOOLS)
},
raise_on_error=False,
timeout=REPRODUCE_TOOL_TIMEOUT)
PROCESSED_TESTCASE_IDS[testcase_id] = True
return return_code
def test_not_capture(self):
"""Test not capture."""
self.popen.returncode = 0
self.popen.communicate.return_value = (None, None)
self.assertIsNone(
process.call('test', cwd='path', env={'NEW': '2'}, capture=False))
self.mock.Popen.assert_called_once_with('test',
shell=True,
cwd='path',
env={
'TEST': '1',
'NEW': '2'
},
stdout=None,
preexec_fn=os.setsid)
self.popen.communicate.assert_called_once_with()
self.mock.store_last_pid.assert_called_once_with(123)
self.assert_exact_calls(self.mock.kill_last_pid, [mock.call()] * 2)
def run_testcase(testcase_id, opts):
"""Attempts to reproduce a testcase."""
return_code, _ = process.call(
('%s reproduce %s %s' % (BINARY_LOCATION, testcase_id, opts)).strip(),
cwd=HOME,
env={
'CF_QUIET': '1',
'USER': '******',
'CHROMIUM_SRC': CHROMIUM_SRC,
'GOMA_GCE_SERVICE_ACCOUNT': 'default',
'PATH': '%s:%s' % (os.environ['PATH'], DEPOT_TOOLS)
},
raise_on_error=False
)
# If the return code is retriable, we don't store it in
# PROCESSED_TESTCASE_IDS. This means the testcase will be run again in the
# next batch.
if return_code not in RETRIABLE_RETURN_CODES:
PROCESSED_TESTCASE_IDS.add(testcase_id)
return return_code
def clean():
"""Clean the repo and all of the sub repos. This combination seems to
work. Please be careful when changing it."""
process.call('git clean -ffdd', cwd=CHROMIUM_SRC)
process.call('git reset --hard', cwd=CHROMIUM_SRC)
# We've encountered the case where `git clean` and `git reset --hard` don't
# clean up the untracked files. I have no idea why. Tracking the files and
# resetting seems to work.
# See: https://github.com/google/clusterfuzz-tools/issues/414
process.call('git add --all', cwd=CHROMIUM_SRC)
process.call('git reset --hard', cwd=CHROMIUM_SRC)
# This fixes the problem in:
# https://github.com/google/clusterfuzz-tools/issues/426
process.call('git checkout origin/master -f', cwd=CHROMIUM_SRC)
# Manually reset dirs.
for path in CLEAN_CHROMIUM_SUBDIRS:
process.call('rm -rf %s' % path, cwd=CHROMIUM_SRC)
process.call('git checkout origin/master %s -f' % path,
cwd=CHROMIUM_SRC)
# --reset resets all uncommitted changes in the sub repo.
process.call(
'gclient sync --reset',
cwd=CHROMIUM_SRC,
env={'PATH': '%s:%s' % (os.environ['PATH'], DEPOT_TOOLS)},
)
def get_supported_jobtypes():
"""Returns a hash of supported job types."""
_, out = process.call(build_command('supported_job_types'), capture=True)
result = yaml.load(out)
result.pop('Version', None)
return result
def get_binary_version():
"""Returns the version of the binary."""
_, out = process.call(build_command('supported_job_types'), capture=True)
return yaml.load(out)['Version']
