def test_copy_directory_to_worker(self, mock_copy_file_to_worker,
mock_remove_directory):
"""Test file_host.copy_directory_to_worker."""
mock_copy_file_to_worker.return_value = True
self.fs.create_file('/host/dir/file1')
self.fs.create_file('/host/dir/file2')
self.fs.create_file('/host/dir/dir2/file3')
self.fs.create_file('/host/dir/dir2/file4')
self.fs.create_file('/host/dir/dir2/dir3/file5')
self.assertTrue(
file_host.copy_directory_to_worker('/host/dir',
'/worker/copied_dir'))
mock_copy_file_to_worker.assert_has_calls([
mock.call('/host/dir/file1', '/worker/copied_dir/file1'),
mock.call('/host/dir/file2', '/worker/copied_dir/file2'),
mock.call('/host/dir/dir2/file3', '/worker/copied_dir/dir2/file3'),
mock.call('/host/dir/dir2/file4', '/worker/copied_dir/dir2/file4'),
mock.call('/host/dir/dir2/dir3/file5',
'/worker/copied_dir/dir2/dir3/file5'),
],
any_order=True)
self.assertTrue(
file_host.copy_directory_to_worker('/host/dir',
'/worker/copied_dir',
replace=True))
mock_copy_file_to_worker.assert_has_calls([
mock.call('/host/dir/file1', '/worker/copied_dir/file1'),
mock.call('/host/dir/file2', '/worker/copied_dir/file2'),
mock.call('/host/dir/dir2/file3', '/worker/copied_dir/dir2/file3'),
mock.call('/host/dir/dir2/file4', '/worker/copied_dir/dir2/file4'),
mock.call('/host/dir/dir2/dir3/file5',
'/worker/copied_dir/dir2/dir3/file5'),
],
any_order=True)
mock_remove_directory.assert_called_with('/worker/copied_dir',
recreate=True)
mock_copy_file_to_worker.return_value = False
self.assertFalse(
file_host.copy_directory_to_worker('/host/dir',
'/worker/copied_dir2'))
def test_copy_directory_to_worker_replace(self):
"""Tests remote copy_directory_to_worker (replacing old dir)"""
src_dir = os.path.join(self.tmp_dir, 'src_dir')
os.mkdir(src_dir)
with open(os.path.join(src_dir, 'file1'), 'w') as f:
f.write('1')
dest_dir = os.path.join(self.tmp_dir, 'dst_dir')
os.mkdir(dest_dir)
with open(os.path.join(dest_dir, 'old_file'), 'w') as f:
f.write('old')
self.assertTrue(
file_host.copy_directory_to_worker(src_dir, dest_dir, replace=True))
self.assert_dirs_equal(src_dir, dest_dir)
def test_copy_directory_to_worker(self):
"""Tests remote copy_directory_to_worker."""
src_dir = os.path.join(self.tmp_dir, 'src_dir')
nested_src_dir = os.path.join(self.tmp_dir, 'src_dir', 'nested')
os.mkdir(src_dir)
os.mkdir(nested_src_dir)
with open(os.path.join(src_dir, 'file1'), 'w') as f:
f.write('1')
with open(os.path.join(nested_src_dir, 'file2'), 'w') as f:
f.write('2')
dest_dir = os.path.join(self.tmp_dir, 'dst_dir')
os.mkdir(dest_dir)
old_file_path = os.path.join(dest_dir, 'old_file')
with open(old_file_path, 'w') as f:
f.write('old')
self.assertTrue(file_host.copy_directory_to_worker(src_dir, dest_dir))
self.assertTrue(os.path.exists(old_file_path))
os.remove(old_file_path)
self.assert_dirs_equal(src_dir, dest_dir)
def update_fuzzer_and_data_bundles(fuzzer_name):
"""Update the fuzzer with a given name if necessary."""
fuzzer = data_types.Fuzzer.query(data_types.Fuzzer.name == fuzzer_name).get()
if not fuzzer:
logs.log_error('No fuzzer exists with name %s.' % fuzzer_name)
raise errors.InvalidFuzzerError
# Set some helper environment variables.
fuzzer_directory = get_fuzzer_directory(fuzzer_name)
environment.set_value('FUZZER_DIR', fuzzer_directory)
environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content)
# If the fuzzer generates large testcases or a large number of small ones
# that don't fit on tmpfs, then use the larger disk directory.
if fuzzer.has_large_testcases:
testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK')
environment.set_value('FUZZ_INPUTS', testcase_disk_directory)
# Adjust the test timeout, if user has provided one.
if fuzzer.timeout:
environment.set_value('TEST_TIMEOUT', fuzzer.timeout)
# Increase fuzz test timeout if the fuzzer timeout is higher than its
# current value.
fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT')
if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout:
environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout)
# Adjust the max testcases if this fuzzer has specified a lower limit.
max_testcases = environment.get_value('MAX_TESTCASES')
if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases:
environment.set_value('MAX_TESTCASES', fuzzer.max_testcases)
# Check for updates to this fuzzer.
version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name)
if (not fuzzer.builtin and
revisions.needs_update(version_file, fuzzer.revision)):
logs.log('Fuzzer update was found, updating.')
# Clear the old fuzzer directory if it exists.
if not shell.remove_directory(fuzzer_directory, recreate=True):
logs.log_error('Failed to clear fuzzer directory.')
return None
# Copy the archive to local disk and unpack it.
archive_path = os.path.join(fuzzer_directory, fuzzer.filename)
if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path):
logs.log_error('Failed to copy fuzzer archive.')
return None
try:
archive.unpack(archive_path, fuzzer_directory)
except Exception:
error_message = ('Failed to unpack fuzzer archive %s '
'(bad archive or unsupported format).') % fuzzer.filename
logs.log_error(error_message)
fuzzer_logs.upload_script_log(
'Fatal error: ' + error_message, fuzzer_name=fuzzer_name)
return None
fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path)
if not os.path.exists(fuzzer_path):
error_message = ('Fuzzer executable %s not found. '
'Check fuzzer configuration.') % fuzzer.executable_path
logs.log_error(error_message)
fuzzer_logs.upload_script_log(
'Fatal error: ' + error_message, fuzzer_name=fuzzer_name)
return None
# Make fuzzer executable.
os.chmod(fuzzer_path, 0o750)
# Cleanup unneeded archive.
shell.remove_file(archive_path)
# Save the current revision of this fuzzer in a file for later checks.
revisions.write_revision_to_revision_file(version_file, fuzzer.revision)
logs.log('Updated fuzzer to revision %d.' % fuzzer.revision)
_clear_old_data_bundles_if_needed()
# Setup data bundles associated with this fuzzer.
data_bundles = ndb_utils.get_all_from_query(
data_types.DataBundle.query(
data_types.DataBundle.name == fuzzer.data_bundle_name))
for data_bundle in data_bundles:
if not update_data_bundle(fuzzer, data_bundle):
return None
# Setup environment variable for launcher script path.
if fuzzer.launcher_script:
fuzzer_launcher_path = os.path.join(fuzzer_directory,
fuzzer.launcher_script)
environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path)
# For launcher script usecase, we need the entire fuzzer directory on the
# worker.
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
worker_fuzzer_directory = file_host.rebase_to_worker_root(
fuzzer_directory)
file_host.copy_directory_to_worker(
fuzzer_directory, worker_fuzzer_directory, replace=True)
return fuzzer
