def set_stats(self, actual_duration, new_units_generated, new_units_added, corpus_size, fuzzing_strategies, fuzzer_stderr, afl_fuzz_output=''): """Create a dict of statistics that can be uploaded to ClusterFuzz and save it in self.stats. Args: actual_duration: The length of time afl-fuzz was run for, in seconds. new_units_generated: The number of new corpus files generated by afl-fuzz. new_units_added: The number of new corpus files left after minimizing those generated by afl-fuzz. fuzzing_strategies: Fuzzing strategies used by AFL. afl_fuzz_output: Output from afl-fuzz. Returns: The stats dictionary. Any values that could not be found default to 0. """ # TODO(metzman): Add expected_duration to stats. # TODO(metzman): Add the other stats that are less clear how to add. assert actual_duration >= 0 assert new_units_generated >= 0 assert new_units_added >= 0 assert new_units_added <= new_units_generated # Set stats passed to this function as arguments. self.stats['actual_duration'] = int(actual_duration) self.stats['new_units_generated'] = new_units_generated self.stats['new_units_added'] = new_units_added self.stats['corpus_size'] = corpus_size # Set log_lines_unwanted stat from parsing fuzzer stderr. self.stats['log_lines_unwanted'] = self._get_unwanted_log_line_count( fuzzer_stderr) # Set dictionary stats if self.dict_path is set. if self.dict_path is not None: self.stats['dict_used'] = 1 self.stats['manual_dict_size'], _ = ( dictionary_manager.get_stats_for_dictionary_file( self.dict_path)) # Read and parse stats from AFL's afl_stats. Then use them to set and # calculate our own stats. self.set_afl_stats() for afl_stat, clusterfuzz_stat in self.AFL_STATS_MAPPING.iteritems(): self.stats[clusterfuzz_stat] = self.get_afl_stat(afl_stat) try: self.stats['average_exec_per_sec'] = int( self.get_afl_stat('execs_done') / actual_duration) except ZeroDivisionError: # Fail gracefully if actual_duration is 0. self.stats['average_exec_per_sec'] = 0 logs.log_error('actual_duration is 0 in fuzzer_stats. ' 'average_exec_per_sec defaulting to 0.') # Normalize |timeout_count| and |crash_count| to be either 0 or 1. for stat_variable in ['crash_count', 'timeout_count']: self.stats[stat_variable] = int(bool(self.stats[stat_variable])) self.set_strategy_stats(fuzzing_strategies) self.set_output_stats(afl_fuzz_output) return self.stats
def parse_performance_features(log_lines, strategies, arguments): """Extract stats for performance analysis.""" # Initialize stats with default values. stats = { 'bad_instrumentation': 0, 'corpus_crash_count': 0, 'corpus_size': 0, 'crash_count': 0, 'dict_used': 0, 'edge_coverage': 0, 'edges_total': 0, 'feature_coverage': 0, 'initial_edge_coverage': 0, 'initial_feature_coverage': 0, 'leak_count': 0, 'log_lines_unwanted': 0, 'log_lines_from_engine': 0, 'log_lines_ignored': 0, 'max_len': 0, 'manual_dict_size': 0, 'merge_edge_coverage': 0, 'new_edges': 0, 'new_features': 0, 'oom_count': 0, 'recommended_dict_size': 0, 'slow_unit_count': 0, 'slow_units_count': 0, 'startup_crash_count': 1, 'timeout_count': 0, } # Extract strategy selection method. stats['strategy_selection_method'] = environment.get_value( 'STRATEGY_SELECTION_METHOD', default_value='default') # Initialize all strategy stats as disabled by default. for strategy_type in strategy.strategy_list: stats[strategy_column_name(strategy_type.name)] = 0 # Process fuzzing strategies used. stats.update(parse_fuzzing_strategies(log_lines, strategies)) (stats['log_lines_unwanted'], stats['log_lines_from_engine'], stats['log_lines_ignored']) = calculate_log_lines(log_lines) # Extract '-max_len' value from arguments, if possible. stats['max_len'] = int( fuzzer_utils.extract_argument( arguments, constants.MAX_LEN_FLAG, remove=False) or stats['max_len']) # Extract sizes of manual and recommended dictionary used for fuzzing. dictionary_path = fuzzer_utils.extract_argument( arguments, constants.DICT_FLAG, remove=False) stats['manual_dict_size'], stats['recommended_dict_size'] = ( dictionary_manager.get_stats_for_dictionary_file(dictionary_path)) # Different crashes and other flags extracted via regexp match. has_corpus = False libfuzzer_inited = False for line in log_lines: if LIBFUZZER_BAD_INSTRUMENTATION_REGEX.match(line): stats['bad_instrumentation'] = 1 continue if LIBFUZZER_CRASH_TESTCASE_REGEX.match(line): stats['crash_count'] = 1 continue if LIBFUZZER_LOG_DICTIONARY_REGEX.match(line): stats['dict_used'] = 1 continue if LEAK_TESTCASE_REGEX.match(line): stats['leak_count'] = 1 continue if (LIBFUZZER_OOM_TESTCASE_REGEX.match(line) or stack_analyzer.OUT_OF_MEMORY_REGEX.match(line)): stats['oom_count'] = 1 continue if LIBFUZZER_SLOW_UNIT_TESTCASE_REGEX.match(line): # Use |slow_unit_count| to track if this run had any slow units at all. # and use |slow_units_count| to track the actual number of slow units in # this run (used by performance analyzer). stats['slow_unit_count'] = 1 stats['slow_units_count'] += 1 continue match = LIBFUZZER_LOG_SEED_CORPUS_INFO_REGEX.match(line) if match: has_corpus = True match = LIBFUZZER_MODULES_LOADED_REGEX.match(line) if match: stats['startup_crash_count'] = 0 stats['edges_total'] = int(match.group(2)) match = LIBFUZZER_LOG_START_INITED_REGEX.match(line) if match: stats['initial_edge_coverage'] = stats['edge_coverage'] = int( match.group(1)) stats['initial_feature_coverage'] = stats['feature_coverage'] = int( match.group(2)) libfuzzer_inited = True continue # This regexp will match multiple lines and will be overwriting the stats. # This is done on purpose, as the last line in the log may have different # format, e.g. 'DONE' without a crash and 'NEW' or 'pulse' with a crash. # Also, ignore values before INITED i.e. while seed corpus is being read. match = LIBFUZZER_LOG_COVERAGE_REGEX.match(line) if match and libfuzzer_inited: stats['edge_coverage'] = int(match.group(1)) stats['feature_coverage'] = int(match.group(2)) continue if (LIBFUZZER_TIMEOUT_TESTCASE_REGEX.match(line) or stack_analyzer.LIBFUZZER_TIMEOUT_REGEX.match(line)): stats['timeout_count'] = 1 continue if not stats['max_len']: # Get "max_len" value from the log, if it has not been found in arguments. match = LIBFUZZER_LOG_MAX_LEN_REGEX.match(line) if match: stats['max_len'] = int(match.group(1)) continue if has_corpus and not stats['log_lines_from_engine']: stats['corpus_crash_count'] = 1 # new_cov_* is a reliable metric when corpus subset strategy is not used. if not stats['strategy_corpus_subset']: assert stats['edge_coverage'] >= stats['initial_edge_coverage'] stats['new_edges'] = ( stats['edge_coverage'] - stats['initial_edge_coverage']) assert stats['feature_coverage'] >= stats['initial_feature_coverage'] stats['new_features'] = ( stats['feature_coverage'] - stats['initial_feature_coverage']) return stats
def parse_performance_features(log_lines, strategies, arguments): """Extract stats for performance analysis.""" # Initialize stats with default values. stats = { 'bad_instrumentation': 0, 'corpus_crash_count': 0, 'corpus_size': 0, 'crash_count': 0, 'dict_used': 0, 'edge_coverage': 0, 'edges_total': 0, 'feature_coverage': 0, 'initial_edge_coverage': 0, 'initial_feature_coverage': 0, 'leak_count': 0, 'log_lines_unwanted': 0, 'log_lines_from_engine': 0, 'log_lines_ignored': 0, 'max_len': 0, 'manual_dict_size': 0, 'merge_edge_coverage': 0, 'merge_new_files': 0, 'merge_new_features': 0, 'oom_count': 0, 'recommended_dict_size': 0, 'slow_unit_count': 0, 'slow_units_count': 0, 'startup_crash_count': 1, 'strategy_corpus_mutations_radamsa': 0, 'strategy_corpus_mutations_ml_rnn': 0, 'strategy_corpus_subset': 0, 'strategy_fork': 0, 'strategy_mutator_plugin': 0, 'strategy_random_max_len': 0, 'strategy_recommended_dict': 0, 'strategy_value_profile': 0, 'timeout_count': 0, } # Process fuzzing strategies used. stats.update(parse_fuzzing_strategies(log_lines, strategies)) # Corpus rss is only applicable when using the full corpus and not # in the corpus subset strategy run. if not stats['strategy_corpus_subset']: stats['corpus_rss_mb'] = 0 (stats['log_lines_unwanted'], stats['log_lines_from_engine'], stats['log_lines_ignored']) = calculate_log_lines(log_lines) # Extract '-max_len' value from arguments, if possible. stats['max_len'] = int( fuzzer_utils.extract_argument( arguments, constants.MAX_LEN_FLAG, remove=False) or stats['max_len']) # Extract sizes of manual and recommended dictionary used for fuzzing. dictionary_path = fuzzer_utils.extract_argument( arguments, constants.DICT_FLAG, remove=False) stats['manual_dict_size'], stats['recommended_dict_size'] = ( dictionary_manager.get_stats_for_dictionary_file(dictionary_path)) # Different crashes and other flags extracted via regexp match. has_corpus = False for line in log_lines: if LIBFUZZER_BAD_INSTRUMENTATION_REGEX.match(line): stats['bad_instrumentation'] = 1 continue if LIBFUZZER_CRASH_TESTCASE_REGEX.match(line): stats['crash_count'] = 1 continue if LIBFUZZER_LOG_DICTIONARY_REGEX.match(line): stats['dict_used'] = 1 continue if LEAK_TESTCASE_REGEX.match(line): stats['leak_count'] = 1 continue if (LIBFUZZER_OOM_TESTCASE_REGEX.match(line) or stack_analyzer.OUT_OF_MEMORY_REGEX.match(line)): stats['oom_count'] = 1 continue if LIBFUZZER_SLOW_UNIT_TESTCASE_REGEX.match(line): # Use |slow_unit_count| to track if this run had any slow units at all. # and use |slow_units_count| to track the actual number of slow units in # this run (used by performance analyzer). stats['slow_unit_count'] = 1 stats['slow_units_count'] += 1 continue match = LIBFUZZER_LOG_SEED_CORPUS_INFO_REGEX.match(line) if match: has_corpus = True if not stats['strategy_corpus_subset']: stats['corpus_rss_mb'] = int(match.group(2)) match = LIBFUZZER_MODULES_LOADED_REGEX.match(line) if match: stats['startup_crash_count'] = 0 stats['edges_total'] = int(match.group(2)) match = LIBFUZZER_LOG_START_INITED_REGEX.match(line) if match: stats['initial_edge_coverage'] = int(match.group(1)) stats['initial_feature_coverage'] = int(match.group(2)) continue # This regexp will match multiple lines and will be overwriting the stats. # This is done on purpose, as the last line in the log may have different # format, e.g. 'DONE' without a crash and 'NEW' or 'pulse' with a crash. match = LIBFUZZER_LOG_COVERAGE_REGEX.match(line) if match: stats['edge_coverage'] = int(match.group(1)) stats['feature_coverage'] = int(match.group(2)) continue if (LIBFUZZER_TIMEOUT_TESTCASE_REGEX.match(line) or stack_analyzer.LIBFUZZER_TIMEOUT_REGEX.match(line)): stats['timeout_count'] = 1 continue if not stats['max_len']: # Get "max_len" value from the log, if it has not been found in arguments. match = LIBFUZZER_LOG_MAX_LEN_REGEX.match(line) if match: stats['max_len'] = int(match.group(1)) continue if has_corpus and not stats['log_lines_from_engine']: stats['corpus_crash_count'] = 1 return stats
def parse_performance_features(log_lines, strategies, arguments): """Extract stats for performance analysis.""" # TODO(ochang): Remove include_strategies once refactor is complete. # Initialize stats with default values. stats = { 'bad_instrumentation': 0, 'corpus_crash_count': 0, 'corpus_size': 0, 'crash_count': 0, 'dict_used': 0, 'edge_coverage': 0, 'edges_total': 0, 'feature_coverage': 0, 'initial_edge_coverage': 0, 'initial_feature_coverage': 0, 'leak_count': 0, 'log_lines_unwanted': 0, 'log_lines_from_engine': 0, 'log_lines_ignored': 0, 'max_len': 0, 'manual_dict_size': 0, 'merge_edge_coverage': 0, 'new_edges': 0, 'new_features': 0, 'oom_count': 0, 'recommended_dict_size': 0, 'slow_unit_count': 0, 'slow_units_count': 0, 'startup_crash_count': 1, 'timeout_count': 0, } # Extract strategy selection method. # TODO(ochang): Move to more general place? stats['strategy_selection_method'] = environment.get_value( 'STRATEGY_SELECTION_METHOD', default_value='default') # Initialize all strategy stats as disabled by default. for strategy_type in strategy.LIBFUZZER_STRATEGY_LIST: stats[strategy_column_name(strategy_type.name)] = 0 # Process fuzzing strategies used. stats.update(parse_fuzzing_strategies(log_lines, strategies)) (stats['log_lines_unwanted'], stats['log_lines_from_engine'], stats['log_lines_ignored']) = calculate_log_lines(log_lines) if stats['log_lines_from_engine'] > 0: stats['startup_crash_count'] = 0 # Extract '-max_len' value from arguments, if possible. stats['max_len'] = int( fuzzer_utils.extract_argument( arguments, constants.MAX_LEN_FLAG, remove=False) or stats['max_len']) # Extract sizes of manual and recommended dictionary used for fuzzing. dictionary_path = fuzzer_utils.extract_argument(arguments, constants.DICT_FLAG, remove=False) stats['manual_dict_size'], stats['recommended_dict_size'] = ( dictionary_manager.get_stats_for_dictionary_file(dictionary_path)) # Different crashes and other flags extracted via regexp match. has_corpus = False for line in log_lines: if LIBFUZZER_BAD_INSTRUMENTATION_REGEX.match(line): stats['bad_instrumentation'] = 1 continue if LIBFUZZER_CRASH_TESTCASE_REGEX.match(line): stats['crash_count'] = 1 continue if LIBFUZZER_LOG_DICTIONARY_REGEX.match(line): stats['dict_used'] = 1 continue if LEAK_TESTCASE_REGEX.match(line): stats['leak_count'] = 1 continue if (LIBFUZZER_OOM_TESTCASE_REGEX.match(line) or stack_analyzer.OUT_OF_MEMORY_REGEX.match(line)): stats['oom_count'] = 1 continue if LIBFUZZER_SLOW_UNIT_TESTCASE_REGEX.match(line): # Use |slow_unit_count| to track if this run had any slow units at all. # and use |slow_units_count| to track the actual number of slow units in # this run (used by performance analyzer). stats['slow_unit_count'] = 1 stats['slow_units_count'] += 1 continue match = LIBFUZZER_LOG_SEED_CORPUS_INFO_REGEX.match(line) if match: has_corpus = True match = LIBFUZZER_MODULES_LOADED_REGEX.match(line) if match: stats['startup_crash_count'] = 0 stats['edges_total'] = int(match.group(2)) if (LIBFUZZER_TIMEOUT_TESTCASE_REGEX.match(line) or stack_analyzer.LIBFUZZER_TIMEOUT_REGEX.match(line)): stats['timeout_count'] = 1 continue if not stats['max_len']: # Get "max_len" value from the log, if it has not been found in arguments. match = LIBFUZZER_LOG_MAX_LEN_REGEX.match(line) if match: stats['max_len'] = int(match.group(1)) continue if has_corpus and not stats['log_lines_from_engine']: stats['corpus_crash_count'] = 1 return stats