def download_artifact_if_needed(build_id, artifact_directory,
artifact_archive_path,
targets_with_type_and_san, artifact_file_name,
output_filename_override, build_params,
build_params_check_path):
"""Downloads artifact to actifacts_archive_path if needed"""
# Check if we already have the symbols in cache.
cached_build_params = utils.read_data_from_file(build_params_check_path,
eval_data=True)
if cached_build_params and cached_build_params == build_params:
# No work to do, same system symbols already in cache.
return
# Delete existing symbols directory first.
shell.remove_directory(artifact_directory, recreate=True)
# Fetch symbol file from cloud storage cache (if available).
found_in_cache = storage.get_file_from_cache_if_exists(
artifact_archive_path, update_modification_time_on_access=False)
if not found_in_cache:
for target_with_type_and_san in targets_with_type_and_san:
# Fetch the artifact now.
fetch_artifact.get(build_id, target_with_type_and_san,
artifact_file_name, artifact_directory,
output_filename_override)
if os.path.exists(artifact_archive_path):
break
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
os.environ['BOT_TMPDIR'] = os.path.join(self.tmp_dir, 'bot_tmpdir')
test_helpers.patch(self, [
'clusterfuzz._internal.datastore.data_handler.'
'get_data_bundle_bucket_name',
'clusterfuzz._internal.system.environment.'
'set_environment_parameters_from_file',
])
test_helpers.patch_environ(self)
# Our tests write data/logs into subdirs of ROOT_DIR. Pivot the ROOT_DIR to
# a temporary one.
new_root = _create_test_root()
os.environ['ROOT_DIR'] = new_root
self.saved_cwd = os.getcwd()
os.chdir(new_root)
environment.set_bot_environment()
fuzz_inputs = os.environ['FUZZ_INPUTS']
shell.remove_directory(fuzz_inputs, recreate=True)
worker_fuzz_inputs = file_host.rebase_to_worker_root(fuzz_inputs)
shell.remove_directory(worker_fuzz_inputs, recreate=True)
environment.set_value('GSUTIL_PATH', os.path.dirname(_which('gsutil')))
test_utils.setup_pubsub('test-clusterfuzz')
test_utils.create_pubsub_topic(pubsub.PubSubClient(),
'test-clusterfuzz',
'jobs-project-linux')
def remove_binding(self, binding):
"""Overriden version of remove_binding that ensures the bound directory is
removed. This is necessary because unlike in regular minijails, we do not
delete the entire chroot directory."""
abs_path = os.path.join(self._chroot_dir, binding.dest_path[1:])
shell.remove_directory(abs_path)
super(ChromeOSChroot, self).remove_binding(binding)
def clean_temp_directories():
"""Clean temporary directories."""
for temp_directory in TEMP_DIRECTORIES:
temp_directory_full_path = os.path.abspath(
os.path.expandvars(temp_directory))
shell.remove_directory(temp_directory_full_path,
recreate=True,
ignore_errors=True)
def test_unpack_file_with_cwd_prefix(self):
"""Test unpack with trusted=False passes with file having './' prefix."""
tgz_path = os.path.join(TESTDATA_PATH, 'cwd-prefix.tgz')
output_directory = tempfile.mkdtemp(prefix='cwd-prefix')
archive.unpack(tgz_path, output_directory, trusted=False)
test_file_path = os.path.join(output_directory, 'test')
self.assertTrue(os.path.exists(test_file_path))
self.assertEqual(open(test_file_path).read(), 'abc\n')
shell.remove_directory(output_directory)
def update_tests_if_needed():
"""Updates layout tests every day."""
data_directory = environment.get_value('FUZZ_DATA')
error_occured = False
expected_task_duration = 60 * 60 # 1 hour.
retry_limit = environment.get_value('FAIL_RETRIES')
temp_archive = os.path.join(data_directory, 'temp.zip')
tests_url = environment.get_value('WEB_TESTS_URL')
# Check if we have a valid tests url.
if not tests_url:
return
# Layout test updates are usually disabled to speedup local testing.
if environment.get_value('LOCAL_DEVELOPMENT'):
return
# |UPDATE_WEB_TESTS| env variable can be used to control our update behavior.
if not environment.get_value('UPDATE_WEB_TESTS'):
return
last_modified_time = persistent_cache.get_value(
TESTS_LAST_UPDATE_KEY, constructor=datetime.datetime.utcfromtimestamp)
if (last_modified_time is not None
and not dates.time_has_expired(last_modified_time,
days=TESTS_UPDATE_INTERVAL_DAYS)):
return
logs.log('Updating layout tests.')
tasks.track_task_start(tasks.Task('update_tests', '', ''),
expected_task_duration)
# Download and unpack the tests archive.
for _ in range(retry_limit):
try:
shell.remove_directory(data_directory, recreate=True)
storage.copy_file_from(tests_url, temp_archive)
archive.unpack(temp_archive, data_directory, trusted=True)
shell.remove_file(temp_archive)
error_occured = False
break
except:
logs.log_error(
'Could not retrieve and unpack layout tests archive. Retrying.'
)
error_occured = True
if not error_occured:
persistent_cache.set_value(TESTS_LAST_UPDATE_KEY,
time.time(),
persist_across_reboots=True)
tasks.track_task_end()
def _download_testcase(testcase_id, testcase, configuration):
"""Download the test case and return its path."""
print('Downloading testcase...')
testcase_download_url = '{url}?id={id}'.format(
url=configuration.get('testcase_download_url'), id=testcase_id)
response, content = http_utils.request(testcase_download_url,
method=http_utils.GET_METHOD,
configuration=configuration)
if response.status != 200:
raise errors.ReproduceToolUnrecoverableError(
'Unable to download test case.')
bot_absolute_filename = response[FILENAME_RESPONSE_HEADER]
# Store the test case in the config directory for debuggability.
testcase_directory = os.path.join(CONFIG_DIRECTORY, 'current-testcase')
shell.remove_directory(testcase_directory, recreate=True)
environment.set_value('FUZZ_INPUTS', testcase_directory)
testcase_path = os.path.join(testcase_directory,
os.path.basename(bot_absolute_filename))
utils.write_data_to_file(content, testcase_path)
# Unpack the test case if it's archived.
# TODO(mbarbella): Rewrite setup.unpack_testcase and share this code.
if testcase.minimized_keys and testcase.minimized_keys != 'NA':
mask = data_types.ArchiveStatus.MINIMIZED
else:
mask = data_types.ArchiveStatus.FUZZED
if testcase.archive_state & mask:
archive.unpack(testcase_path, testcase_directory)
file_list = archive.get_file_list(testcase_path)
testcase_path = None
for file_name in file_list:
if os.path.basename(file_name) == os.path.basename(
testcase.absolute_path):
testcase_path = os.path.join(testcase_directory, file_name)
break
if not testcase_path:
raise errors.ReproduceToolUnrecoverableError(
'Test case file was not found in archive.\n'
'Original filename: {absolute_path}.\n'
'Archive contents: {file_list}'.format(
absolute_path=testcase.absolute_path, file_list=file_list))
return testcase_path
def test_remove_shutil_success(self):
"""Test remove with shutil."""
self.mock.exists.side_effect = [True, True, False]
self.assertTrue(shell.remove_directory('dir'))
self.mock.system.assert_has_calls(
[mock.call('rm -rf "dir" > /dev/null 2>&1')])
self.mock.rmtree.assert_has_calls([mock.call('dir', onerror=mock.ANY)])
def download_artifact_if_needed(
build_id, artifact_directory, artifact_archive_path,
targets_with_type_and_san, artifact_file_name, output_filename_override):
"""Downloads artifact to actifacts_archive_path if needed"""
# Delete existing symbols directory first.
shell.remove_directory(artifact_directory, recreate=True)
# Fetch symbol file from cloud storage cache (if available).
found_in_cache = storage.get_file_from_cache_if_exists(
artifact_archive_path, update_modification_time_on_access=False)
if not found_in_cache:
for target_with_type_and_san in targets_with_type_and_san:
# Fetch the artifact now.
fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name,
artifact_directory, output_filename_override)
if os.path.exists(artifact_archive_path):
break
def test_remove_shutil_failure_ignore_errors(self):
self.mock.exists.side_effect = [True, True, True]
self.assertFalse(shell.remove_directory('dir', ignore_errors=True))
self.mock.log_warn.assert_has_calls(
[mock.call('Failed to clear directory dir.')])
self.assertEqual(0, self.mock.log_error.call_count)
self.mock.system.assert_has_calls(
[mock.call('rm -rf "dir" > /dev/null 2>&1')])
self.mock.rmtree.assert_has_calls([mock.call('dir', onerror=mock.ANY)])
def _clear_old_data_bundles_if_needed():
"""Clear old data bundles so as to keep the disk cache restricted to
|_DATA_BUNDLE_CACHE_COUNT| data bundles and prevent potential out-of-disk
spaces."""
data_bundles_directory = environment.get_value('DATA_BUNDLES_DIR')
dirs = []
for filename in os.listdir(data_bundles_directory):
file_path = os.path.join(data_bundles_directory, filename)
if not os.path.isdir(file_path):
continue
dirs.append(file_path)
dirs_to_remove = sorted(dirs, key=os.path.getmtime,
reverse=True)[_DATA_BUNDLE_CACHE_COUNT:]
for dir_to_remove in dirs_to_remove:
logs.log(
'Removing data bundle directory to keep disk cache small: %s' %
dir_to_remove)
shell.remove_directory(dir_to_remove)
def download_latest_build(build_info, image_regexes, image_directory):
"""Download the latest build artifact for the given branch and target."""
# Check if our local build matches the latest build. If not, we will
# download it.
build_id = build_info['bid']
target = build_info['target']
last_build_info = persistent_cache.get_value(constants.LAST_FLASH_BUILD_KEY)
if last_build_info and last_build_info['bid'] == build_id:
return
# Clean up the images directory first.
shell.remove_directory(image_directory, recreate=True)
for image_regex in image_regexes:
image_file_path = fetch_artifact.get(build_id, target, image_regex,
image_directory)
if not image_file_path:
logs.log_error('Failed to download artifact %s for '
'branch %s and target %s.' %
(image_file_path, build_info['branch'], target))
return
if image_file_path.endswith('.zip') or image_file_path.endswith('.tar.gz'):
archive.unpack(image_file_path, image_directory)
def prepare_model_directory(fuzzer_name):
"""Prepare model directory, and return model path.
Args:
fuzzer_name: Name of the fuzzer to which this model belongs.
Returns:
Model path. For example, if `/tmp/model` is the directory containing model
files(e.g. rnn.index), the path should be '/tmp/model/rnn'.
"""
# Get temporary directory.
temp_directory = environment.get_value('BOT_TMPDIR')
# Create model directory.
model_directory = os.path.join(temp_directory, fuzzer_name)
shell.remove_directory(model_directory, recreate=True)
if not download_model_from_gcs(model_directory, fuzzer_name):
return None
# Got the model. Return model path.
return os.path.join(model_directory, constants.RNN_MODEL_NAME)
def test_remove_shutil_onerror(self):
"""Test shutil invoking onerror."""
self.mock.exists.side_effect = [True, True, False]
self.assertTrue(shell.remove_directory('dir'))
self.mock.system.assert_has_calls(
[mock.call('rm -rf "dir" > /dev/null 2>&1')])
self.mock.rmtree.assert_has_calls([mock.call('dir', onerror=mock.ANY)])
onerror = self.mock.rmtree.call_args[1]['onerror']
fake_fn = mock.MagicMock()
fake_fn.side_effect = OSError()
onerror(fake_fn, 'dir/child', ImportError())
self.mock.chmod.assert_has_calls([mock.call('dir/child', 0o750)])
fake_fn.assert_has_calls([mock.call('dir/child')])
def _test_remove_os_specific(self, platform, recreate,
raise_makedirs_error):
"""Helper for testing removing dir with os-specific command."""
self.mock.platform.return_value = platform
self.mock.exists.side_effect = [True, False, False]
if raise_makedirs_error:
self.mock.makedirs.side_effect = OSError()
result = shell.remove_directory('dir', recreate=recreate)
if recreate:
self.assertEqual(not raise_makedirs_error, result)
else:
self.assertTrue(result)
self.mock.rmtree.assert_has_calls([])
if recreate:
self.mock.makedirs.assert_has_calls([mock.call('dir')])
else:
self.mock.makedirs.assert_has_calls([])
def do_corpus_pruning(context, last_execution_failed, revision):
"""Run corpus pruning."""
# Set |FUZZ_TARGET| environment variable to help with unarchiving only fuzz
# target and its related files.
environment.set_value('FUZZ_TARGET', context.fuzz_target.binary)
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import tasks_host
return tasks_host.do_corpus_pruning(context, last_execution_failed,
revision)
if not build_manager.setup_build(revision=revision):
raise CorpusPruningException('Failed to setup build.')
build_directory = environment.get_value('BUILD_DIR')
start_time = datetime.datetime.utcnow()
runner = Runner(build_directory, context)
pruner = CorpusPruner(runner)
fuzzer_binary_name = os.path.basename(runner.target_path)
# If our last execution failed, shrink to a randomized corpus of usable size
# to prevent corpus from growing unbounded and recurring failures when trying
# to minimize it.
if last_execution_failed:
for corpus_url in [
context.corpus.get_gcs_url(),
context.quarantine_corpus.get_gcs_url()
]:
_limit_corpus_size(corpus_url)
# Get initial corpus to process from GCS.
context.sync_to_disk()
initial_corpus_size = shell.get_directory_file_count(
context.initial_corpus_path)
# Restore a small batch of quarantined units back to corpus.
context.restore_quarantined_units()
# Shrink to a minimized corpus using corpus merge.
pruner_stats = pruner.run(context.initial_corpus_path,
context.minimized_corpus_path,
context.bad_units_path)
# Sync minimized corpus back to GCS.
context.sync_to_gcs()
# Create corpus backup.
# Temporarily copy the past crash regressions folder into the minimized corpus
# so that corpus backup archive can have both.
regressions_input_dir = os.path.join(context.initial_corpus_path,
'regressions')
regressions_output_dir = os.path.join(context.minimized_corpus_path,
'regressions')
if shell.get_directory_file_count(regressions_input_dir):
shutil.copytree(regressions_input_dir, regressions_output_dir)
backup_bucket = environment.get_value('BACKUP_BUCKET')
corpus_backup_url = corpus_manager.backup_corpus(
backup_bucket, context.corpus, context.minimized_corpus_path)
shell.remove_directory(regressions_output_dir)
minimized_corpus_size_units = shell.get_directory_file_count(
context.minimized_corpus_path)
minimized_corpus_size_bytes = shell.get_directory_size(
context.minimized_corpus_path)
logs.log('Corpus pruned from %d to %d units.' %
(initial_corpus_size, minimized_corpus_size_units))
# Process bad units found during merge.
# Mapping of crash state -> CorpusCrash
crashes = {}
pruner.process_bad_units(context.bad_units_path,
context.quarantine_corpus_path, crashes)
context.quarantine_corpus.rsync_from_disk(context.quarantine_corpus_path)
# Store corpus stats into CoverageInformation entity.
project_qualified_name = context.fuzz_target.project_qualified_name()
today = datetime.datetime.utcnow().date()
coverage_info = data_types.CoverageInformation(
fuzzer=project_qualified_name, date=today)
quarantine_corpus_size = shell.get_directory_file_count(
context.quarantine_corpus_path)
quarantine_corpus_dir_size = shell.get_directory_size(
context.quarantine_corpus_path)
# Save the minimize corpus size before cross pollination to put in BigQuery.
pre_pollination_corpus_size = minimized_corpus_size_units
# Populate coverage stats.
coverage_info.corpus_size_units = minimized_corpus_size_units
coverage_info.corpus_size_bytes = minimized_corpus_size_bytes
coverage_info.quarantine_size_units = quarantine_corpus_size
coverage_info.quarantine_size_bytes = quarantine_corpus_dir_size
coverage_info.corpus_backup_location = corpus_backup_url
coverage_info.corpus_location = context.corpus.get_gcs_url()
coverage_info.quarantine_location = context.quarantine_corpus.get_gcs_url()
# Calculate remaining time to use for shared corpus merging.
time_remaining = _get_time_remaining(start_time)
if time_remaining <= 0:
logs.log_warn('Not enough time for shared corpus merging.')
return None
cross_pollinator = CrossPollinator(runner)
pollinator_stats = cross_pollinator.run(time_remaining)
context.sync_to_gcs()
# Update corpus size stats.
minimized_corpus_size_units = shell.get_directory_file_count(
context.minimized_corpus_path)
minimized_corpus_size_bytes = shell.get_directory_size(
context.minimized_corpus_path)
coverage_info.corpus_size_units = minimized_corpus_size_units
coverage_info.corpus_size_bytes = minimized_corpus_size_bytes
logs.log('Finished.')
sources = ','.join([
fuzzer.fuzz_target.project_qualified_name()
for fuzzer in context.cross_pollinate_fuzzers
])
cross_pollination_stats = None
if pruner_stats and pollinator_stats:
cross_pollination_stats = CrossPollinationStats(
project_qualified_name, context.cross_pollination_method, sources,
context.tag, initial_corpus_size, pre_pollination_corpus_size,
pruner_stats['edge_coverage'], pollinator_stats['edge_coverage'],
pruner_stats['feature_coverage'],
pollinator_stats['feature_coverage'])
return CorpusPruningResult(coverage_info=coverage_info,
crashes=list(crashes.values()),
fuzzer_binary_name=fuzzer_binary_name,
revision=environment.get_value('APP_REVISION'),
cross_pollination_stats=cross_pollination_stats)
def unpack_crash_testcases(crash_testcases_directory):
"""Unpacks the old crash testcases in the provided directory."""
for testcase in ndb_utils.get_all_from_model(data_types.Testcase):
testcase_id = testcase.key.id()
# 1. If we have already stored the testcase, then just skip.
if testcase_id in STORED_TESTCASES_LIST:
continue
# 2. Make sure that it is a unique crash testcase. Ignore duplicates,
# uploaded repros.
if testcase.status != 'Processed':
continue
# 3. Check if the testcase is fixed. If not, skip.
if testcase.open:
continue
# 4. Check if the testcase has a minimized repro. If not, skip.
if not testcase.minimized_keys or testcase.minimized_keys == 'NA':
continue
# 5. Only use testcases that have bugs associated with them.
if not testcase.bug_information:
continue
# 6. Existing IPC testcases are un-interesting and unused in further
# mutations. Due to size bloat, ignoring these for now.
if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION):
continue
# 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests).
if archive.get_archive_type(testcase.absolute_path):
continue
# 8. Skip in-process fuzzer testcases, since these are only applicable to
# fuzz targets and don't run with blackbox binaries.
if testcase.fuzzer_name and testcase.fuzzer_name in ['afl', 'libFuzzer']:
continue
# Un-pack testcase.
try:
_, input_directory, _ = setup.unpack_testcase(testcase)
except Exception:
logs.log_error('Failed to unpack testcase %d.' % testcase.key.id())
continue
# Move this to our crash testcases directory.
crash_testcase_directory = os.path.join(crash_testcases_directory,
str(testcase_id))
shell.move(input_directory, crash_testcase_directory)
# Re-create input directory for unpacking testcase in next iteration.
shell.create_directory(input_directory)
STORED_TESTCASES_LIST.append(testcase_id)
# Remove testcase directories that exceed the max size limit.
for directory_name in os.listdir(crash_testcases_directory):
directory_path = os.path.join(crash_testcases_directory, directory_name)
if not os.path.isdir(directory_path):
continue
if shell.get_directory_size(directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE:
continue
shell.remove_directory(directory_path)
# Rename all fuzzed testcase files as regular files.
for root, _, files in os.walk(crash_testcases_directory):
for filename in files:
if not filename.startswith(testcase_manager.FUZZ_PREFIX):
continue
file_path = os.path.join(root, filename)
stripped_file_name = os.path.basename(file_path)[len(
testcase_manager.FUZZ_PREFIX):]
stripped_file_path = os.path.join(
os.path.dirname(file_path), stripped_file_name)
try:
os.rename(file_path, stripped_file_path)
except:
raise Exception('Failed to rename testcase %s.' % file_path)
# Remove empty files and dirs to avoid the case where a fuzzer randomly
# chooses an empty dir/file and generates zero testcases.
shell.remove_empty_files(crash_testcases_directory)
shell.remove_empty_directories(crash_testcases_directory)
def tearDown(self):
adb.remove_directory(self.device_temp_dir)
shell.remove_directory(self.local_temp_dir)
def recreate_directory(directory_path):
"""Delete directory if exists, create empty directory. Throw an exception if
either fails."""
if not shell.remove_directory(directory_path, recreate=True):
raise Exception('Failed to recreate directory: ' + directory_path)
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# If the fuzzer generates large testcases or a large number of small ones
# that don't fit on tmpfs, then use the larger disk directory.
if fuzzer.has_large_testcases:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin
and revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return None
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return None
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = (
'Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return None
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = (
'Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log('Fatal error: ' + error_message,
fuzzer_name=fuzzer_name)
return None
# Make fuzzer executable.
os.chmod(fuzzer_path, 0o750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file,
fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
_clear_old_data_bundles_if_needed()
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return None
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = os.path.join(fuzzer_directory,
fuzzer.launcher_script)
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
# For launcher script usecase, we need the entire fuzzer directory on the
# worker.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
worker_fuzzer_directory = file_host.rebase_to_worker_root(
fuzzer_directory)
file_host.copy_directory_to_worker(fuzzer_directory,
worker_fuzzer_directory,
replace=True)
return fuzzer
def tearDown(self): shell.remove_directory(self.model_directory) shell.remove_directory(self.log_directory)
def remove_created_dirs(self, chroot_dir, minijail_created_dirs=None):
if minijail_created_dirs is None:
minijail_created_dirs = ['tmp', 'proc', 'dev']
for directory in minijail_created_dirs:
shell.remove_directory(os.path.join(chroot_dir, directory))
def remove_directory(request, _):
"""Remove a directory."""
result = shell.remove_directory(request.path, request.recreate)
return untrusted_runner_pb2.RemoveDirectoryResponse(result=result)
def _cleanup():
"""Clean up after running the tool."""
temp_directory = environment.get_value('ROOT_DIR')
assert 'tmp' in temp_directory
shell.remove_directory(temp_directory)
def tearDown(self):
shell.remove_directory(self.build_directory)
def execute_task(full_fuzzer_name, job_type):
"""Execute ML RNN training task.
The task is training RNN model by default. If more models are developed,
arguments can be modified to specify which model to use.
Args:
fuzzer_name: Name of fuzzer, e.g. libpng_read_fuzzer.
job_type: Job type, e.g. libfuzzer_chrome_asan.
"""
del job_type
# Sets up fuzzer binary build.
fuzz_target = data_handler.get_fuzz_target(full_fuzzer_name)
if not fuzz_target:
logs.log_warn(
f'Fuzzer not found: {full_fuzzer_name}, skip RNN training.')
return
fuzzer_name = fuzz_target.project_qualified_name()
# Directory to place training files, such as logs, models, corpus.
# Use |FUZZ_INPUTS_DISK| since it is not size constrained.
temp_directory = environment.get_value('FUZZ_INPUTS_DISK')
# Get corpus.
corpus_directory = get_corpus_directory(temp_directory, fuzzer_name)
shell.remove_directory(corpus_directory, recreate=True)
logs.log('Downloading corpus backup for %s.' % fuzzer_name)
if not ml_train_utils.get_corpus(corpus_directory, fuzzer_name):
logs.log_error('Failed to download corpus backup for %s.' %
fuzzer_name)
return
# Get the directory to save models.
model_directory = get_model_files_directory(temp_directory, fuzzer_name)
shell.remove_directory(model_directory, recreate=True)
# Get the directory to save training logs.
log_directory = get_model_log_directory(temp_directory, fuzzer_name)
shell.remove_directory(log_directory, recreate=True)
result = train_rnn(corpus_directory, model_directory, log_directory)
# Training process exited abnormally but not caused by timeout, meaning
# error occurred during execution.
if result.return_code and not result.timed_out:
if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL:
logs.log_warn(
'ML RNN training task for fuzzer %s aborted due to small corpus.'
% fuzzer_name)
else:
logs.log_error(
'ML RNN training task for fuzzer %s failed with ExitCode = %d.'
% (fuzzer_name, result.return_code),
output=utils.decode_to_unicode(result.output))
return
# Timing out may be caused by large training corpus, but intermediate models
# are frequently saved and can be uploaded.
if result.timed_out:
logs.log_warn('ML RNN training task for %s timed out.' % fuzzer_name)
upload_model_to_gcs(model_directory, fuzzer_name)
def cleanup():
"""Clean up temporary metadata."""
shell.remove_directory(get_temp_dir())
def cleanup(self):
"""Cleanup state."""
for path in self._created_directories:
shell.remove_directory(path)
def tearDown(self):
shell.remove_directory(self.output_directory)
shell.remove_directory(self.empty_directory)
def tearDown(self):
shell.remove_directory(self.temp_directory)
