def cleanse(self, target_path, arguments, input_path, output_path,
max_time):
"""Optional (but recommended): Cleanse a testcase.
Args:
target_path: Path to the target.
arguments: Additional arguments needed for testcase cleanse.
input_path: Path to the reproducer input.
output_path: Path to the cleansed output.
max_time: Maximum allowed time for the cleanse.
Returns:
A ReproduceResult.
Raises:
TimeoutError: If the cleanse exceeds max_time.
"""
runner = libfuzzer.get_runner(target_path)
libfuzzer.set_sanitizer_options(target_path)
cleanse_tmp_dir = self._create_temp_corpus_dir('cleanse-workdir')
result = runner.cleanse_crash(input_path,
output_path,
max_time,
artifact_prefix=cleanse_tmp_dir,
additional_args=arguments)
if result.timed_out:
raise engine.TimeoutError('Cleanse timed out\n' + result.output)
return engine.ReproduceResult(result.command, result.return_code,
result.time_executed, result.output)
def reproduce(self, target_path, input_path, arguments, max_time):
"""Reproduce a crash given an input.
Args:
target_path: Path to the target.
input_path: Path to the reproducer input.
arguments: Additional arguments needed for reproduction.
max_time: Maximum allowed time for the reproduction.
Returns:
A ReproduceResult.
Raises:
TimeoutError: If the reproduction exceeds max_time.
"""
runner = libfuzzer.get_runner(target_path)
libfuzzer.set_sanitizer_options(target_path)
# Remove fuzzing specific arguments. This is only really needed for legacy
# testcases, and can be removed in the distant future.
arguments = arguments[:]
libfuzzer.remove_fuzzing_arguments(arguments)
runs_argument = constants.RUNS_FLAG + str(constants.RUNS_TO_REPRODUCE)
arguments.append(runs_argument)
result = runner.run_single_testcase(
input_path, timeout=max_time, additional_args=arguments)
if result.timed_out:
raise engine.TimeoutError('Reproducing timed out\n' + result.output)
return engine.ReproduceResult(result.command, result.return_code,
result.time_executed,
utils.decode_to_unicode(result.output))
def wrapped(*args, **kwargs):
"""Wrapper for adding retry logic."""
for retry_attempt in range(num_retries + 1):
# Wait for channel to (re)connect if necessary.
state = _check_channel_state(config.RECONNECT_TIMEOUT_SECONDS)
if state == ChannelState.INCONSISTENT:
# No point retrying if the worker is inconsistent.
monitoring_metrics.HOST_INCONSISTENT_COUNT.increment()
logs.log_warn('Worker got into an inconsistent state.')
host_exit_no_return(return_code=0)
if state == ChannelState.NOT_READY:
# Channel still isn't ready.
logs.log_warn(
'Channel failed to become ready within reconnect timeout.')
if retry_attempt == num_retries:
# Last attempt.
host_exit_no_return()
continue
try:
return func(*args, **kwargs)
except grpc.RpcError as e:
# For timeouts, which aren't fatal errors, resurface the right
# exception.
# TODO(mbarbella): Ignoring errors on the next line fixes an issue while
# trying to support this code in both Python 2 and 3, but may not be
# necessary in Python 3 since presumably the exception class will
# allow us to properly convert it to a string. Delete after migrating.
exception_message = e.message.decode('utf-8', errors='ignore')
if 'TimeoutError' in exception_message:
# TODO(ochang): Replace with generic TimeoutError in Python 3.
raise engine.TimeoutError(e.message)
if num_retries == 0:
# Just re-raise the original exception if this RPC is not configured
# for retries.
raise
logs.log_warn('Failed RPC: ' + exception_message)
if retry_attempt == num_retries:
# Last attempt.
host_exit_no_return()
time.sleep(RPC_FAIL_WAIT_TIME)
def minimize_corpus(self, target_path, arguments, input_dirs, output_dir,
reproducers_dir, max_time):
"""Optional (but recommended): run corpus minimization.
Args:
target_path: Path to the target.
arguments: Additional arguments needed for corpus minimization.
input_dirs: Input corpora.
output_dir: Output directory to place minimized corpus.
reproducers_dir: The directory to put reproducers in when crashes are
found.
max_time: Maximum allowed time for the minimization.
Returns:
A Result object.
Raises:
TimeoutError: If the corpus minimization exceeds max_time.
Error: If the merge failed in some other way.
"""
runner = libfuzzer.get_runner(target_path)
libfuzzer.set_sanitizer_options(target_path)
merge_tmp_dir = self._create_temp_corpus_dir("merge-workdir")
result = runner.merge(
[output_dir] + input_dirs,
merge_timeout=max_time,
tmp_dir=merge_tmp_dir,
additional_args=arguments,
artifact_prefix=reproducers_dir,
merge_control_file=getattr(self, "_merge_control_file", None),
)
if result.timed_out:
raise engine.TimeoutError("Merging new testcases timed out\n" +
result.output)
if result.return_code != 0:
raise MergeError("Merging new testcases failed: " + result.output)
merge_stats = stats.parse_stats_from_merge_log(
result.output.splitlines())
# TODO(ochang): Get crashes found during merge.
return engine.FuzzResult(result.output, result.command, [],
merge_stats, result.time_executed)
def wrapped(*args, **kwargs):
"""Wrapper for adding retry logic."""
for retry_attempt in range(num_retries + 1):
# Wait for channel to (re)connect if necessary.
state = _check_channel_state(config.RECONNECT_TIMEOUT_SECONDS)
if state == ChannelState.INCONSISTENT:
# No point retrying if the worker is inconsistent.
monitoring_metrics.HOST_INCONSISTENT_COUNT.increment()
logs.log_warn('Worker got into an inconsistent state.')
host_exit_no_return(return_code=0)
if state == ChannelState.NOT_READY:
# Channel still isn't ready.
logs.log_warn(
'Channel failed to become ready within reconnect timeout.')
if retry_attempt == num_retries:
# Last attempt.
host_exit_no_return()
continue
try:
return func(*args, **kwargs)
except grpc.RpcError as e:
# For timeouts, which aren't fatal errors, resurface the right
# exception.
if 'TimeoutError' in repr(e):
# TODO(ochang): Replace with generic TimeoutError in Python 3.
# Use __str__ to avoid newstrs. TODO(ochang): Use plain str() once
# migrated to Python 3.
raise engine.TimeoutError(e.__str__())
if num_retries == 0:
# Just re-raise the original exception if this RPC is not configured
# for retries.
raise
logs.log_warn('Failed RPC: ' + repr(e))
if retry_attempt == num_retries:
# Last attempt.
host_exit_no_return()
time.sleep(RPC_FAIL_WAIT_TIME)
def _minimize_corpus_two_step(self, target_path, arguments,
existing_corpus_dirs, new_corpus_dir,
output_corpus_dir, reproducers_dir,
max_time):
"""Optional (but recommended): run corpus minimization.
Args:
target_path: Path to the target.
arguments: Additional arguments needed for corpus minimization.
existing_corpus_dirs: Input corpora that existed before the fuzzing run.
new_corpus_dir: Input corpus that was generated during the fuzzing run.
Must have at least one new file.
output_corpus_dir: Output directory to place minimized corpus.
reproducers_dir: The directory to put reproducers in when crashes are
found.
max_time: Maximum allowed time for the minimization.
Returns:
A Result object.
"""
if not _is_multistep_merge_supported(target_path):
# Fallback to the old single step merge. It does not support incremental
# stats and provides only `edge_coverage` and `feature_coverage` stats.
logs.log(
'Old version of libFuzzer is used. Using single step merge.')
return self.minimize_corpus(
target_path, arguments,
existing_corpus_dirs + [new_corpus_dir], output_corpus_dir,
reproducers_dir, max_time)
# The dir where merge control file is located must persist for both merge
# steps. The second step re-uses the MCF produced during the first step.
merge_control_file_dir = self._create_temp_corpus_dir('mcf_tmp_dir')
self._merge_control_file = os.path.join(merge_control_file_dir, 'MCF')
# Two step merge process to obtain accurate stats for the new corpus units.
# See https://reviews.llvm.org/D66107 for a more detailed description.
merge_stats = {}
# Step 1. Use only existing corpus and collect "initial" stats.
result_1 = self.minimize_corpus(target_path, arguments,
existing_corpus_dirs,
output_corpus_dir, reproducers_dir,
max_time)
merge_stats['initial_edge_coverage'] = result_1.stats['edge_coverage']
merge_stats['initial_feature_coverage'] = result_1.stats[
'feature_coverage']
# Clear the output dir as it does not have any new units at this point.
engine_common.recreate_directory(output_corpus_dir)
# Adjust the time limit for the time we spent on the first merge step.
max_time -= result_1.time_executed
if max_time <= 0:
raise engine.TimeoutError('Merging new testcases timed out\n' +
result_1.logs)
# Step 2. Process the new corpus units as well.
result_2 = self.minimize_corpus(
target_path, arguments, existing_corpus_dirs + [new_corpus_dir],
output_corpus_dir, reproducers_dir, max_time)
merge_stats['edge_coverage'] = result_2.stats['edge_coverage']
merge_stats['feature_coverage'] = result_2.stats['feature_coverage']
# Diff the stats to obtain accurate values for the new corpus units.
merge_stats['new_edges'] = (merge_stats['edge_coverage'] -
merge_stats['initial_edge_coverage'])
merge_stats['new_features'] = (merge_stats['feature_coverage'] -
merge_stats['initial_feature_coverage'])
output = result_1.logs + '\n\n' + result_2.logs
if (merge_stats['new_edges'] < 0 or merge_stats['new_features'] < 0):
logs.log_error('Two step merge failed.',
merge_stats=merge_stats,
output=output)
merge_stats['new_edges'] = 0
merge_stats['new_features'] = 0
self._merge_control_file = None
# TODO(ochang): Get crashes found during merge.
return engine.FuzzResult(
output, result_2.command, [], merge_stats,
result_1.time_executed + result_2.time_executed)