def remove_testcases_from_directories(directories):
"""Removes all testcases and their dependencies from testcase directories."""
generators = []
for directory in directories:
if not directory.strip():
continue
# If there is a bot-specific files list, delete it now.
bot_testcases_file_path = utils.get_bot_testcases_file_path(directory)
shell.remove_file(bot_testcases_file_path)
generators.append(shell.walk(directory))
for generator in generators:
for structure in generator:
base_directory = structure[0]
for filename in structure[2]:
if not is_testcase_resource(filename):
continue
if filename.startswith(RESOURCES_PREFIX):
# In addition to removing this file, remove all resources.
resources_file_path = os.path.join(base_directory,
filename)
resources = read_resource_list(resources_file_path)
for resource in resources:
shell.remove_file(resource)
file_path = os.path.join(base_directory, filename)
shell.remove_file(file_path)
def get_testcases_from_directories(directories):
"""Returns all testcases from testcase directories."""
testcase_paths = []
max_testcases = environment.get_value('MAX_TESTCASES')
generators = []
for directory in directories:
if not directory.strip():
continue
generators.append(shell.walk(directory))
for generator in generators:
for structure in generator:
base_directory = structure[0]
for filename in structure[2]:
if not filename.startswith(FUZZ_PREFIX):
continue
if filename.endswith(COVERAGE_SUFFIX):
continue
file_path = os.path.join(base_directory, filename)
if not os.path.getsize(file_path):
continue
testcase_paths.append(utils.normalize_path(file_path))
if len(testcase_paths) == max_testcases:
return testcase_paths
return testcase_paths
def get_resource_dependencies(testcase_absolute_path, test_prefix=FUZZ_PREFIX):
"""Returns the list of testcase resource dependencies."""
resources = []
if not os.path.exists(testcase_absolute_path):
return resources
base_directory = os.path.dirname(testcase_absolute_path)
testcase_filename = os.path.basename(testcase_absolute_path)
# FIXME(mbarbella): Remove this when all fuzzers are using "resources-".
# This code includes the dependencies that begin with
# dependency prefix and are referenced in the testcase.
testcase_contents = None
for filename in os.listdir(base_directory):
if filename.startswith(DEPENDENCY_PREFIX):
# Only load the testcase contents if necessary.
if not testcase_contents:
with open(testcase_absolute_path, 'rb') as file_handle:
testcase_contents = file_handle.read()
if filename.encode('utf-8') in testcase_contents:
file_path = os.path.join(base_directory, filename)
resources.append(file_path)
# This code includes the dependencies in cases when the testcase itself is a
# just a wrapper file around the actual testcase.
if DEPENDENCY_PREFIX in testcase_absolute_path:
dependency_filename = os.path.splitext(testcase_filename)[0]
dependency_filename = re.compile(DEPENDENCY_PREFIX).sub(
'', dependency_filename, 1)
dependency_filename = re.compile(FUZZ_PREFIX).sub(
'', dependency_filename, 1)
dependency_filename = re.compile(HTTP_PREFIX).sub(
'', dependency_filename, 1)
dependency_file_path = os.path.join(base_directory,
dependency_filename)
resources.append(dependency_file_path)
# Check to see if this test case lists all resources in a resources file.
if testcase_filename.startswith(test_prefix):
stripped_testcase_name = testcase_filename[len(test_prefix):]
resources_filename = '%s%s' % (RESOURCES_PREFIX,
stripped_testcase_name)
resources_file_path = os.path.join(base_directory, resources_filename)
resources += read_resource_list(resources_file_path)
# For extensions, archive everything in the extension directory.
if APPS_PREFIX in testcase_filename or EXTENSIONS_PREFIX in testcase_filename:
for root, _, files in shell.walk(base_directory):
for filename in files:
file_path = os.path.join(root, filename)
if file_path == testcase_absolute_path:
continue
resources.append(file_path)
return resources
def clear_old_files(directory, extracted_file_set):
"""Remove files from the directory that isn't in the given file list."""
for root_directory, _, filenames in shell.walk(directory):
for filename in filenames:
file_path = os.path.join(root_directory, filename)
if file_path not in extracted_file_set:
shell.remove_file(file_path)
shell.remove_empty_directories(directory)
def clear_pyc_files(directory):
"""Recursively remove all .pyc files from the given directory"""
for root_directory, _, filenames in shell.walk(directory):
for filename in filenames:
if not filename.endswith('.pyc'):
continue
file_path = os.path.join(root_directory, filename)
shell.remove_file(file_path)
def get_fuzz_targets_local(path):
"""Get list of fuzz targets paths (local)."""
fuzz_target_paths = []
for root, _, files in shell.walk(path):
for filename in files:
file_path = os.path.join(root, filename)
if is_fuzz_target_local(file_path):
fuzz_target_paths.append(file_path)
return fuzz_target_paths
def list_files(request, _):
"""List files."""
file_paths = []
if request.recursive:
for root, _, files in shell.walk(request.path):
for filename in files:
file_paths.append(os.path.join(root, filename))
else:
file_paths.extend(
os.path.join(request.path, path)
for path in os.listdir(request.path))
return untrusted_runner_pb2.ListFilesResponse(file_paths=file_paths)
def copy_directory_to_worker(host_directory, worker_directory, replace=False):
"""Recursively copy a directory to the worker. Directories are created as
needed. Unless |replace| is True, files already in |worker_path| will remain
after this call."""
if replace:
remove_directory(worker_directory, recreate=True)
for root, _, files in shell.walk(host_directory):
for filename in files:
file_path = os.path.join(root, filename)
worker_file_path = os.path.join(
worker_directory, os.path.relpath(file_path, host_directory))
if not copy_file_to_worker(file_path, worker_file_path):
logs.log_warn('Failed to copy %s to worker.' % file_path)
return False
return True
def create_testcase_list_file(output_directory):
"""Create a testcase list file for tests in a directory."""
files_list = []
files_list_file_path = os.path.join(output_directory,
TESTCASE_LIST_FILENAME)
for root, _, files in shell.walk(output_directory):
for filename in files:
if filename.endswith(INFO_FILE_EXTENSION):
# Skip an info file.
continue
file_path = os.path.join(root, filename)
if not utils.is_valid_testcase_file(file_path,
check_if_exists=False):
continue
normalized_relative_file_path = utils.get_normalized_relative_path(
file_path, output_directory)
files_list.append(normalized_relative_file_path)
utils.write_data_to_file('\n'.join(sorted(files_list)),
files_list_file_path)
def find_fuzzer_path(build_directory, fuzzer_name):
"""Find the fuzzer path with the given name."""
if not build_directory:
# Grey-box fuzzers might not have the build directory for a particular job
# configuration when doing variant task testing (e.g. Android on-device
# fuzz target might not exist on host). In this case, treat it similar to
# target not found by returning None.
logs.log_warn('No build directory found for fuzzer: %s' % fuzzer_name)
return None
if environment.platform() == 'FUCHSIA':
# Fuchsia targets are not on disk.
return fuzzer_name
if environment.is_android_kernel():
return os.path.join(build_directory, 'syzkaller', 'bin', 'syz-manager')
# TODO(ochang): This is necessary for legacy testcases, which include the
# project prefix in arguments. Remove this in the near future.
project_name = environment.get_value('PROJECT_NAME')
legacy_name_prefix = u''
if project_name:
legacy_name_prefix = project_name + u'_'
fuzzer_filename = environment.get_executable_filename(fuzzer_name)
for root, _, files in shell.walk(build_directory):
for filename in files:
if (legacy_name_prefix + filename == fuzzer_name
or filename == fuzzer_filename):
return os.path.join(root, filename)
# This is an expected case when doing regression testing with old builds
# that do not have that fuzz target. It can also happen when a host sends a
# message to an untrusted worker that just restarted and lost information on
# build directory.
logs.log_warn('Fuzzer: %s not found in build_directory: %s.' %
(fuzzer_name, build_directory))
return None
def _list_files_recursive(self, fs_path):
"""List files recursively."""
for root, _, filenames in shell.walk(fs_path):
for filename in filenames:
yield os.path.join(root, filename)