def analyze_and_update_recommended_dictionary(runner, fuzzer_name, log_lines,
corpus_directory, arguments):
"""Extract and analyze recommended dictionary from fuzzer output, then update
the corresponding dictionary stored in GCS if needed."""
logs.log('Extracting and analyzing recommended dictionary for %s.' %
fuzzer_name)
# Extract recommended dictionary elements from the log.
dict_manager = dictionary_manager.DictionaryManager(fuzzer_name)
recommended_dictionary = (
dict_manager.parse_recommended_dictionary_from_log_lines(log_lines))
if not recommended_dictionary:
logs.log('No recommended dictionary in output from %s.' % fuzzer_name)
return None
# Write recommended dictionary into a file and run '-analyze_dict=1'.
temp_dictionary_filename = (fuzzer_name +
dictionary_manager.DICTIONARY_FILE_EXTENSION +
'.tmp')
temp_dictionary_path = os.path.join(fuzzer_utils.get_temp_dir(),
temp_dictionary_filename)
with open(temp_dictionary_path, 'w') as file_handle:
file_handle.write('\n'.join(recommended_dictionary))
dictionary_analysis = runner.analyze_dictionary(
temp_dictionary_path,
corpus_directory,
analyze_timeout=get_dictionary_analysis_timeout(),
additional_args=arguments)
if dictionary_analysis.timed_out:
logs.log_warn('Recommended dictionary analysis for %s timed out.' %
fuzzer_name)
return None
if dictionary_analysis.return_code != 0:
logs.log_warn('Recommended dictionary analysis for %s failed: %d.' %
(fuzzer_name, dictionary_analysis.return_code))
return None
# Extract dictionary elements considered useless, calculate the result.
useless_dictionary = dict_manager.parse_useless_dictionary_from_data(
dictionary_analysis.output)
logs.log(
'%d out of %d recommended dictionary elements for %s are useless.' %
(len(useless_dictionary), len(recommended_dictionary), fuzzer_name))
recommended_dictionary = set(recommended_dictionary) - set(
useless_dictionary)
if not recommended_dictionary:
return None
new_elements_added = dict_manager.update_recommended_dictionary(
recommended_dictionary)
logs.log('Added %d new elements to the recommended dictionary for %s.' %
(new_elements_added, fuzzer_name))
return recommended_dictionary
def test_recommended_dictionary_merge(self):
"""Test merging with GCS copy of recommended dictionary."""
fake_gcs_dict_path = os.path.join(
DATA_DIRECTORY, 'fake_gcs_recommended_dictionary.txt')
dict_manager = dictionary_manager.DictionaryManager('fuzzer_name')
log_data = utils.read_data_from_file(os.path.join(
DATA_DIRECTORY, 'log_with_recommended_dict.txt'),
eval_data=False).decode('utf-8')
dict_from_log = dict_manager.parse_recommended_dictionary_from_data(
log_data)
utils.write_data_to_file('\n'.join(dict_from_log),
self.local_dict_path)
dictionary_manager.merge_dictionary_files(self.local_dict_path,
fake_gcs_dict_path,
self.local_dict_path)
# Compare resulting dictionary with its expected result.
merged_dictionary = self._parse_dictionary_file(self.local_dict_path)
expected_dictionary_path = os.path.join(
DATA_DIRECTORY, 'expected_merged_recommended_dictionary.txt')
expected_dictionary = self._parse_dictionary_file(
expected_dictionary_path)
self.assertEqual(sorted(merged_dictionary),
sorted(expected_dictionary))
def test_useless_dictionary_parse(self):
"""Test parsing of useless dictionary from fuzzer log."""
dict_manager = dictionary_manager.DictionaryManager('fuzzer_name')
log_data = utils.read_data_from_file(
os.path.join(self.data_directory, 'log_with_useless_dict.txt'),
eval_data=False)
useless_dict = dict_manager.parse_useless_dictionary_from_data(log_data)
expected_dictionary_path = os.path.join(
self.data_directory, 'expected_parsed_useless_dictionary.txt')
expected_dictionary = self._parse_dictionary_file(expected_dictionary_path)
self.assertEqual(sorted(useless_dict), sorted(expected_dictionary))
def test_recommended_dictionary_parse(self):
"""Test parsing of recommended dictionary from fuzzer log."""
dict_manager = dictionary_manager.DictionaryManager('fuzzer_name')
log_data = utils.read_data_from_file(
os.path.join(DATA_DIRECTORY, 'log_with_recommended_dict.txt'),
eval_data=False)
recommended_dict = dict_manager.parse_recommended_dictionary_from_data(
log_data)
expected_dictionary_path = os.path.join(
DATA_DIRECTORY, 'expected_parsed_recommended_dictionary.txt')
expected_dictionary = self._parse_dictionary_file(expected_dictionary_path)
self.assertEqual(sorted(recommended_dict), sorted(expected_dictionary))
def add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path):
"""Add recommended dictionary from GCS to existing .dict file or create
a new one and update the arguments as needed.
This function modifies |arguments| list in some cases."""
recommended_dictionary_path = os.path.join(
fuzzer_utils.get_temp_dir(),
dictionary_manager.RECOMMENDED_DICTIONARY_FILENAME)
dict_manager = dictionary_manager.DictionaryManager(fuzzer_name)
try:
# Bail out if cannot download recommended dictionary from GCS.
if not dict_manager.download_recommended_dictionary_from_gcs(
recommended_dictionary_path):
return False
except Exception, ex:
logs.log_error('Exception downloading recommended dictionary:\n%s.' %
str(ex))
return False
def add_recommended_dictionary(arguments, fuzzer_name, fuzzer_path):
"""Add recommended dictionary from GCS to existing .dict file or create
a new one and update the arguments as needed.
This function modifies |arguments| list in some cases."""
recommended_dictionary_path = os.path.join(
fuzzer_utils.get_temp_dir(),
dictionary_manager.RECOMMENDED_DICTIONARY_FILENAME)
dict_manager = dictionary_manager.DictionaryManager(fuzzer_name)
try:
# Bail out if cannot download recommended dictionary from GCS.
if not dict_manager.download_recommended_dictionary_from_gcs(
recommended_dictionary_path):
return False
except Exception as ex:
logs.log_error('Exception downloading recommended dictionary:\n%s.' %
str(ex))
return False
# Bail out if the downloaded dictionary is empty.
if not os.path.getsize(recommended_dictionary_path):
return False
# Check if there is an existing dictionary file in arguments.
original_dictionary_path = fuzzer_utils.extract_argument(
arguments, constants.DICT_FLAG)
merged_dictionary_path = (
original_dictionary_path
or dictionary_manager.get_default_dictionary_path(fuzzer_path))
merged_dictionary_path += MERGED_DICT_SUFFIX
dictionary_manager.merge_dictionary_files(original_dictionary_path,
recommended_dictionary_path,
merged_dictionary_path)
arguments.append(constants.DICT_FLAG + merged_dictionary_path)
return True
