def test_copy_directory_to_worker(self, mock_copy_file_to_worker, mock_remove_directory): """Test file_host.copy_directory_to_worker.""" mock_copy_file_to_worker.return_value = True self.fs.create_file('/host/dir/file1') self.fs.create_file('/host/dir/file2') self.fs.create_file('/host/dir/dir2/file3') self.fs.create_file('/host/dir/dir2/file4') self.fs.create_file('/host/dir/dir2/dir3/file5') self.assertTrue( file_host.copy_directory_to_worker('/host/dir', '/worker/copied_dir')) mock_copy_file_to_worker.assert_has_calls([ mock.call('/host/dir/file1', '/worker/copied_dir/file1'), mock.call('/host/dir/file2', '/worker/copied_dir/file2'), mock.call('/host/dir/dir2/file3', '/worker/copied_dir/dir2/file3'), mock.call('/host/dir/dir2/file4', '/worker/copied_dir/dir2/file4'), mock.call('/host/dir/dir2/dir3/file5', '/worker/copied_dir/dir2/dir3/file5'), ], any_order=True) self.assertTrue( file_host.copy_directory_to_worker('/host/dir', '/worker/copied_dir', replace=True)) mock_copy_file_to_worker.assert_has_calls([ mock.call('/host/dir/file1', '/worker/copied_dir/file1'), mock.call('/host/dir/file2', '/worker/copied_dir/file2'), mock.call('/host/dir/dir2/file3', '/worker/copied_dir/dir2/file3'), mock.call('/host/dir/dir2/file4', '/worker/copied_dir/dir2/file4'), mock.call('/host/dir/dir2/dir3/file5', '/worker/copied_dir/dir2/dir3/file5'), ], any_order=True) mock_remove_directory.assert_called_with('/worker/copied_dir', recreate=True) mock_copy_file_to_worker.return_value = False self.assertFalse( file_host.copy_directory_to_worker('/host/dir', '/worker/copied_dir2'))
def test_copy_directory_to_worker_replace(self): """Tests remote copy_directory_to_worker (replacing old dir)""" src_dir = os.path.join(self.tmp_dir, 'src_dir') os.mkdir(src_dir) with open(os.path.join(src_dir, 'file1'), 'w') as f: f.write('1') dest_dir = os.path.join(self.tmp_dir, 'dst_dir') os.mkdir(dest_dir) with open(os.path.join(dest_dir, 'old_file'), 'w') as f: f.write('old') self.assertTrue( file_host.copy_directory_to_worker(src_dir, dest_dir, replace=True)) self.assert_dirs_equal(src_dir, dest_dir)
def test_copy_directory_to_worker(self): """Tests remote copy_directory_to_worker.""" src_dir = os.path.join(self.tmp_dir, 'src_dir') nested_src_dir = os.path.join(self.tmp_dir, 'src_dir', 'nested') os.mkdir(src_dir) os.mkdir(nested_src_dir) with open(os.path.join(src_dir, 'file1'), 'w') as f: f.write('1') with open(os.path.join(nested_src_dir, 'file2'), 'w') as f: f.write('2') dest_dir = os.path.join(self.tmp_dir, 'dst_dir') os.mkdir(dest_dir) old_file_path = os.path.join(dest_dir, 'old_file') with open(old_file_path, 'w') as f: f.write('old') self.assertTrue(file_host.copy_directory_to_worker(src_dir, dest_dir)) self.assertTrue(os.path.exists(old_file_path)) os.remove(old_file_path) self.assert_dirs_equal(src_dir, dest_dir)
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query(data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # If the fuzzer generates large testcases or a large number of small ones # that don't fit on tmpfs, then use the larger disk directory. if fuzzer.has_large_testcases: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return None # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return None try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ('Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log( 'Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ('Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log( 'Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None # Make fuzzer executable. os.chmod(fuzzer_path, 0o750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) _clear_old_data_bundles_if_needed() # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return None # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = os.path.join(fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) # For launcher script usecase, we need the entire fuzzer directory on the # worker. if environment.is_trusted_host(): from bot.untrusted_runner import file_host worker_fuzzer_directory = file_host.rebase_to_worker_root( fuzzer_directory) file_host.copy_directory_to_worker( fuzzer_directory, worker_fuzzer_directory, replace=True) return fuzzer