def analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines, corpus_directory, arguments): """Extract and analyze recommended dictionary from fuzzer output, then update the corresponding dictionary stored in GCS if needed.""" logs.log('Extracting and analyzing recommended dictionary for %s.' % fuzzer_name) # Extract recommended dictionary elements from the log. dict_manager = dictionary_manager.DictionaryManager(fuzzer_name) recommended_dictionary = ( dict_manager.parse_recommended_dictionary_from_log_lines(log_lines)) if not recommended_dictionary: logs.log('No recommended dictionary in output from %s.' % fuzzer_name) return None # Write recommended dictionary into a file and run '-analyze_dict=1'. temp_dictionary_filename = (fuzzer_name + dictionary_manager.DICTIONARY_FILE_EXTENSION + '.tmp') temp_dictionary_path = os.path.join(fuzzer_utils.get_temp_dir(), temp_dictionary_filename) with open(temp_dictionary_path, 'w') as file_handle: file_handle.write('\n'.join(recommended_dictionary)) dictionary_analysis = runner.analyze_dictionary( temp_dictionary_path, corpus_directory, analyze_timeout=get_dictionary_analysis_timeout(), additional_args=arguments) if dictionary_analysis.timed_out: logs.log_warn('Recommended dictionary analysis for %s timed out.' % fuzzer_name) return None if dictionary_analysis.return_code != 0: logs.log_warn('Recommended dictionary analysis for %s failed: %d.' % (fuzzer_name, dictionary_analysis.return_code)) return None # Extract dictionary elements considered useless, calculate the result. useless_dictionary = dict_manager.parse_useless_dictionary_from_data( dictionary_analysis.output) logs.log( '%d out of %d recommended dictionary elements for %s are useless.' % (len(useless_dictionary), len(recommended_dictionary), fuzzer_name)) recommended_dictionary = set(recommended_dictionary) - set( useless_dictionary) if not recommended_dictionary: return None new_elements_added = dict_manager.update_recommended_dictionary( recommended_dictionary) logs.log('Added %d new elements to the recommended dictionary for %s.' % (new_elements_added, fuzzer_name)) return recommended_dictionary
def test_recommended_dictionary_merge(self): """Test merging with GCS copy of recommended dictionary.""" fake_gcs_dict_path = os.path.join( DATA_DIRECTORY, 'fake_gcs_recommended_dictionary.txt') dict_manager = dictionary_manager.DictionaryManager('fuzzer_name') log_data = utils.read_data_from_file(os.path.join( DATA_DIRECTORY, 'log_with_recommended_dict.txt'), eval_data=False).decode('utf-8') dict_from_log = dict_manager.parse_recommended_dictionary_from_data( log_data) utils.write_data_to_file('\n'.join(dict_from_log), self.local_dict_path) dictionary_manager.merge_dictionary_files(self.local_dict_path, fake_gcs_dict_path, self.local_dict_path) # Compare resulting dictionary with its expected result. merged_dictionary = self._parse_dictionary_file(self.local_dict_path) expected_dictionary_path = os.path.join( DATA_DIRECTORY, 'expected_merged_recommended_dictionary.txt') expected_dictionary = self._parse_dictionary_file( expected_dictionary_path) self.assertEqual(sorted(merged_dictionary), sorted(expected_dictionary))
def test_useless_dictionary_parse(self): """Test parsing of useless dictionary from fuzzer log.""" dict_manager = dictionary_manager.DictionaryManager('fuzzer_name') log_data = utils.read_data_from_file( os.path.join(self.data_directory, 'log_with_useless_dict.txt'), eval_data=False) useless_dict = dict_manager.parse_useless_dictionary_from_data(log_data) expected_dictionary_path = os.path.join( self.data_directory, 'expected_parsed_useless_dictionary.txt') expected_dictionary = self._parse_dictionary_file(expected_dictionary_path) self.assertEqual(sorted(useless_dict), sorted(expected_dictionary))
def test_recommended_dictionary_parse(self): """Test parsing of recommended dictionary from fuzzer log.""" dict_manager = dictionary_manager.DictionaryManager('fuzzer_name') log_data = utils.read_data_from_file( os.path.join(DATA_DIRECTORY, 'log_with_recommended_dict.txt'), eval_data=False) recommended_dict = dict_manager.parse_recommended_dictionary_from_data( log_data) expected_dictionary_path = os.path.join( DATA_DIRECTORY, 'expected_parsed_recommended_dictionary.txt') expected_dictionary = self._parse_dictionary_file(expected_dictionary_path) self.assertEqual(sorted(recommended_dict), sorted(expected_dictionary))
def add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path): """Add recommended dictionary from GCS to existing .dict file or create a new one and update the arguments as needed. This function modifies |arguments| list in some cases.""" recommended_dictionary_path = os.path.join( fuzzer_utils.get_temp_dir(), dictionary_manager.RECOMMENDED_DICTIONARY_FILENAME) dict_manager = dictionary_manager.DictionaryManager(fuzzer_name) try: # Bail out if cannot download recommended dictionary from GCS. if not dict_manager.download_recommended_dictionary_from_gcs( recommended_dictionary_path): return False except Exception, ex: logs.log_error('Exception downloading recommended dictionary:\n%s.' % str(ex)) return False
def add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path): """Add recommended dictionary from GCS to existing .dict file or create a new one and update the arguments as needed. This function modifies |arguments| list in some cases.""" recommended_dictionary_path = os.path.join( fuzzer_utils.get_temp_dir(), dictionary_manager.RECOMMENDED_DICTIONARY_FILENAME) dict_manager = dictionary_manager.DictionaryManager(fuzzer_name) try: # Bail out if cannot download recommended dictionary from GCS. if not dict_manager.download_recommended_dictionary_from_gcs( recommended_dictionary_path): return False except Exception as ex: logs.log_error('Exception downloading recommended dictionary:\n%s.' % str(ex)) return False # Bail out if the downloaded dictionary is empty. if not os.path.getsize(recommended_dictionary_path): return False # Check if there is an existing dictionary file in arguments. original_dictionary_path = fuzzer_utils.extract_argument( arguments, constants.DICT_FLAG) merged_dictionary_path = ( original_dictionary_path or dictionary_manager.get_default_dictionary_path(fuzzer_path)) merged_dictionary_path += MERGED_DICT_SUFFIX dictionary_manager.merge_dictionary_files(original_dictionary_path, recommended_dictionary_path, merged_dictionary_path) arguments.append(constants.DICT_FLAG + merged_dictionary_path) return True