def _unpack_clusterfuzz_seed_corpus(fuzz_target_path, corpus_directory):
"""If a clusterfuzz seed corpus archive is available, unpack it into the
corpus directory if it exists. Copied from unpack_seed_corpus in
engine_common.py in ClusterFuzz.
"""
oss_fuzz_corpus = environment.get('OSS_FUZZ_CORPUS')
if oss_fuzz_corpus:
benchmark = environment.get('BENCHMARK')
corpus_archive_filename = f'{benchmark}.zip'
oss_fuzz_corpus_archive_path = posixpath.join(
experiment_utils.get_oss_fuzz_corpora_filestore_path(),
corpus_archive_filename)
seed_corpus_archive_path = posixpath.join(FUZZ_TARGET_DIR,
corpus_archive_filename)
filestore_utils.cp(oss_fuzz_corpus_archive_path,
seed_corpus_archive_path)
else:
seed_corpus_archive_path = get_clusterfuzz_seed_corpus_path(
fuzz_target_path)
if not seed_corpus_archive_path:
return
with zipfile.ZipFile(seed_corpus_archive_path) as zip_file:
# Unpack seed corpus recursively into the root of the main corpus
# directory.
idx = 0
for seed_corpus_file in zip_file.infolist():
if seed_corpus_file.filename.endswith('/'):
# Ignore directories.
continue
# Allow callers to opt-out of unpacking large files.
if seed_corpus_file.file_size > CORPUS_ELEMENT_BYTES_LIMIT:
continue
output_filename = '%016d' % idx
output_file_path = os.path.join(corpus_directory, output_filename)
zip_file.extract(seed_corpus_file, output_file_path)
idx += 1
logs.info('Unarchived %d files from seed corpus %s.', idx,
seed_corpus_archive_path)
def copy_resources_to_bucket(config_dir: str, config: Dict):
"""Copy resources the dispatcher will need for the experiment to the
experiment_filestore."""
def filter_file(tar_info):
"""Filter out unnecessary directories."""
if FILTER_SOURCE_REGEX.match(tar_info.name):
return None
return tar_info
# Set environment variables to use corresponding filestore_utils.
os.environ['EXPERIMENT_FILESTORE'] = config['experiment_filestore']
os.environ['EXPERIMENT'] = config['experiment']
experiment_filestore_path = experiment_utils.get_experiment_filestore_path(
)
base_destination = os.path.join(experiment_filestore_path, 'input')
# Send the local source repository to the cloud for use by dispatcher.
# Local changes to any file will propagate.
source_archive = 'src.tar.gz'
with tarfile.open(source_archive, 'w:gz') as tar:
tar.add(utils.ROOT_DIR, arcname='', recursive=True, filter=filter_file)
filestore_utils.cp(source_archive, base_destination + '/', parallel=True)
os.remove(source_archive)
# Send config files.
destination = os.path.join(base_destination, 'config')
filestore_utils.rsync(config_dir, destination, parallel=True)
# If |oss_fuzz_corpus| flag is set, copy latest corpora from each benchmark
# (if available) in our filestore bucket.
if config['oss_fuzz_corpus']:
oss_fuzz_corpora_dir = (
experiment_utils.get_oss_fuzz_corpora_filestore_path())
for benchmark in config['benchmarks']:
add_oss_fuzz_corpus(benchmark, oss_fuzz_corpora_dir)