def initialize_resources_dir(): """Download Fuchsia QEMU resources from GCS bucket.""" resources_dir = environment.get_value('RESOURCES_DIR') if not resources_dir: raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR') fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia') shell.create_directory(fuchsia_resources_dir, recreate=True) fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL') if not fuchsia_resources_url: raise errors.FuchsiaConfigError( 'Could not find path for remote' 'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir ] logs.log("Beginning Fuchsia SDK download.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia' 'resources: ' + result.output) logs.log("Fuchsia SDK download complete.") return fuchsia_resources_dir
def test(self): """Tests copy_local_directory_to_remote.""" utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a')) shell.create_directory(os.path.join(self.local_temp_dir, 'b')) utils.write_data_to_file('c', os.path.join(self.local_temp_dir, 'b', 'c')) adb.copy_local_directory_to_remote(self.local_temp_dir, self.device_temp_dir) self.assertTrue(adb.file_exists(os.path.join(self.device_temp_dir, 'a'))) self.assertFalse( adb.directory_exists(os.path.join(self.device_temp_dir, 'a'))) self.assertEqual( adb.get_file_size(os.path.join(self.device_temp_dir, 'a')), 1) self.assertTrue( adb.directory_exists(os.path.join(self.device_temp_dir, 'b'))) self.assertFalse(adb.file_exists(os.path.join(self.device_temp_dir, 'b'))) self.assertTrue( adb.file_exists(os.path.join(self.device_temp_dir, 'b', 'c'))) self.assertFalse( adb.directory_exists(os.path.join(self.device_temp_dir, 'b', 'c'))) self.assertEqual( adb.get_file_size(os.path.join(self.device_temp_dir, 'b', 'c')), 1)
def get_temp_dir(): """Return the temp dir.""" temp_dirname = 'temp-' + str(os.getpid()) temp_directory = os.path.join( environment.get_value('FUZZ_INPUTS_DISK'), temp_dirname) shell.create_directory(temp_directory) return temp_directory
def _http_request(url, body=None, method=_POST_METHOD, force_reauthorization=False): """Make a POST request to the specified URL.""" authorization = _get_authorization(force_reauthorization) headers = { 'User-Agent': 'clusterfuzz-reproduce', 'Authorization': authorization } http = httplib2.Http() request_body = json_utils.dumps(body) if body else '' response, content = http.request(url, method=method, headers=headers, body=request_body) # If the server returns 401 we may need to reauthenticate. Try the request # a second time if this happens. if response.status == 401 and not force_reauthorization: return _http_request(url, body, method=method, force_reauthorization=True) if 'x-clusterfuzz-authorization' in response: shell.create_directory(os.path.dirname(AUTHORIZATION_CACHE_FILE), create_intermediates=True) utils.write_data_to_file(response['x-clusterfuzz-authorization'], AUTHORIZATION_CACHE_FILE) return response, content
def create_merge_directory(): """Create the merge directory and return its path.""" merge_directory_path = get_merge_directory() shell.create_directory(merge_directory_path, create_intermediates=True, recreate=True) return merge_directory_path
def initialize(): """Initialize the persistent cache, creating the directory used to store the values.""" cache_directory_path = environment.get_value('CACHE_DIR') if os.path.exists(cache_directory_path): clear_values() else: shell.create_directory(cache_directory_path, create_intermediates=True)
def _create_temp_corpus_directory(self, name): """Create temporary corpus directory. Returns path to the created directory.""" testcases_directory = environment.get_value("FUZZ_INPUTS_DISK") directory_path = os.path.join(testcases_directory, name) shell.create_directory(directory_path) self._created_directories.append(directory_path) return directory_path
def get_runner(fuzzer_path, temp_dir=None): """Get a libfuzzer runner.""" use_minijail = environment.get_value('USE_MINIJAIL') build_dir = environment.get_value('BUILD_DIR') dataflow_build_dir = environment.get_value('DATAFLOW_BUILD_DIR') if use_minijail: # Set up chroot and runner. if environment.is_chromeos_system_job(): minijail_chroot = minijail.ChromeOSChroot(build_dir) else: minijail_chroot = minijail.MinijailChroot(base_dir=temp_dir) # While it's possible for dynamic binaries to run without this, they need # to be accessible for symbolization etc. For simplicity we bind BUILD_DIR # to the same location within the chroot, which leaks the directory # structure of CF but this shouldn't be a big deal. minijail_chroot.add_binding( minijail.ChrootBinding(build_dir, build_dir, False)) if dataflow_build_dir: minijail_chroot.add_binding( minijail.ChrootBinding(dataflow_build_dir, dataflow_build_dir, False)) # Also bind the build dir to /out to make it easier to hardcode references # to data files. minijail_chroot.add_binding( minijail.ChrootBinding(build_dir, '/out', False)) minijail_bin = os.path.join(minijail_chroot.directory, 'bin') shell.create_directory(minijail_bin) # Set up /bin with llvm-symbolizer to allow symbolized stacktraces. # Don't copy if it already exists (e.g. ChromeOS chroot jail). llvm_symbolizer_source_path = environment.get_llvm_symbolizer_path() llvm_symbolizer_destination_path = os.path.join( minijail_bin, 'llvm-symbolizer') if not os.path.exists(llvm_symbolizer_destination_path): shutil.copy(llvm_symbolizer_source_path, llvm_symbolizer_destination_path) # copy /bin/sh, necessary for system(). if not environment.is_chromeos_system_job(): # The chroot has its own shell we don't need to copy (and probably # shouldn't because of library differences). shutil.copy(os.path.realpath('/bin/sh'), os.path.join(minijail_bin, 'sh')) runner = MinijailLibFuzzerRunner(fuzzer_path, minijail_chroot) elif environment.platform() == 'FUCHSIA': runner = FuchsiaQemuLibFuzzerRunner(fuzzer_path) else: runner = LibFuzzerRunner(fuzzer_path) return runner
def _makedirs(self, directory): """Create directories for binding in chroot. Args: directory: The absolute path to the directory target in the chroot. """ if directory[0] == '/': directory = directory[1:] shell.create_directory(os.path.join(self._chroot_dir, directory), create_intermediates=True)
def init_corpus(self, source_dir, destination_dir): """Uses corpus from the cloud to initialize syzkaller corpus. Args: source_dir: Folder where the corpus is downloaded from the cloud. destination_dir: Folder where syzkaller will be looking for corpus. """ source_file = os.path.join(source_dir, self._get_device_corpus_db_filename()) shell.create_directory(destination_dir) destination_file = os.path.join(destination_dir, CORPUS_DB_FILENAME) if os.path.isfile(source_file): shutil.copy(source_file, destination_file)
def convert_path_for_write(self, remote_path, directory=OBJECTS_DIR): """Get the local FS path for writing to the remote path. Creates any intermediate directories if necessary (except for the parent bucket directory).""" bucket, path = get_bucket_name_and_path(remote_path) if not os.path.exists(self._fs_bucket_path(bucket)): raise RuntimeError( 'Bucket {bucket} does not exist.'.format(bucket=bucket)) fs_path = self._fs_path(bucket, path, directory) shell.create_directory(os.path.dirname(fs_path), create_intermediates=True) return fs_path
def _cross_pollinate_other_fuzzer_corpuses(self): """Add other fuzzer corpuses to shared corpus path for cross-pollination.""" corpus_backup_date = utils.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) for cross_pollinate_fuzzer in self.cross_pollinate_fuzzers: project_qualified_name = ( cross_pollinate_fuzzer.fuzz_target.project_qualified_name()) backup_bucket_name = cross_pollinate_fuzzer.backup_bucket_name corpus_engine_name = cross_pollinate_fuzzer.corpus_engine_name corpus_backup_url = corpus_manager.gcs_url_for_backup_file( backup_bucket_name, corpus_engine_name, project_qualified_name, corpus_backup_date, ) corpus_backup_local_filename = "%s-%s" % ( project_qualified_name, os.path.basename(corpus_backup_url), ) corpus_backup_local_path = os.path.join( self.shared_corpus_path, corpus_backup_local_filename) if not storage.exists(corpus_backup_url, ignore_errors=True): # This can happen in cases when a new fuzz target is checked in or if # missed to capture a backup for a particular day (for OSS-Fuzz, this # will result in a 403 instead of 404 since that GCS path belongs to # other project). So, just log a warning for debugging purposes only. logs.log_warn("Corpus backup does not exist, ignoring: %s." % corpus_backup_url) continue if not storage.copy_file_from(corpus_backup_url, corpus_backup_local_path): continue corpus_backup_output_directory = os.path.join( self.shared_corpus_path, project_qualified_name) shell.create_directory(corpus_backup_output_directory) result = archive.unpack(corpus_backup_local_path, corpus_backup_output_directory) shell.remove_file(corpus_backup_local_path) if result: logs.log( "Corpus backup url %s successfully unpacked into shared corpus." % corpus_backup_url) else: logs.log_error("Failed to unpack corpus backup from url %s." % corpus_backup_url)
def get_corpus_directory(input_directory, project_qualified_name): """Get the corpus directory given a project qualified fuzz target name.""" corpus_directory = os.path.join(input_directory, project_qualified_name) if environment.is_trusted_host(): from bot.untrusted_runner import file_host corpus_directory = file_host.rebase_to_worker_root(corpus_directory) # Create corpus directory if it does not exist already. if environment.is_trusted_host(): from bot.untrusted_runner import file_host file_host.create_directory(corpus_directory, create_intermediates=True) else: shell.create_directory(corpus_directory) return corpus_directory
def _download_testcase(testcase_id, testcase, configuration): """Download the test case and return its path.""" testcase_download_url = '{url}?id={id}'.format( url=configuration.get('testcase_download_url'), id=testcase_id) response, content = http_utils.request(testcase_download_url, method=http_utils.GET_METHOD, configuration=configuration) if response.status != 200: raise errors.ReproduceToolUnrecoverableError( 'Unable to download test case.') # Create a temporary directory where we can store the test case. bot_absolute_filename = response['x-goog-meta-filename'] testcase_directory = os.path.join(environment.get_value('ROOT_DIR'), 'current-testcase') shell.create_directory(testcase_directory) environment.set_value('FUZZ_INPUTS', testcase_directory) testcase_path = os.path.join(testcase_directory, os.path.basename(bot_absolute_filename)) utils.write_data_to_file(content, testcase_path) # Unpack the test case if it's archived. # TODO(mbarbella): Rewrite setup.unpack_testcase and share this code. if testcase.minimized_keys and testcase.minimized_keys != 'NA': mask = data_types.ArchiveStatus.MINIMIZED else: mask = data_types.ArchiveStatus.FUZZED if testcase.archive_state & mask: archive.unpack(testcase_path, testcase_directory) file_list = archive.get_file_list(testcase_path) testcase_path = None for file_name in file_list: if testcase.absolute_path.endswith(file_name): testcase_path = os.path.join(testcase_directory, file_name) break if not testcase_path: raise errors.ReproduceToolUnrecoverableError( 'Test case file was not found in archive.\n' 'Original filename: {absolute_path}.\n' 'Archive contents: {file_list}'.format( absolute_path=testcase.absolute_path, file_list=file_list)) return testcase_path
def initialize_resources_dir(): """Download Fuchsia QEMU resources from GCS bucket.""" # This module depends on multiprocessing, which is not available in # appengine, and since appengine *imports* this file (but does not run this # function!), we import it here. from google_cloud_utils import gsutil resources_dir = environment.get_value('RESOURCES_DIR') if not resources_dir: raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR') fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia') shell.create_directory( fuchsia_resources_dir, create_intermediates=True, recreate=True) # Bucket for QEMU resources. fuchsia_resources_url = environment.get_value('FUCHSIA_RESOURCES_URL') if not fuchsia_resources_url: raise errors.FuchsiaConfigError( 'Could not find path for remote' 'Fuchsia resources bucket (FUCHSIA_RESOURCES_URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir ] logs.log("Beginning Fuchsia SDK download.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia ' 'resources: ' + result.output) logs.log("Fuchsia SDK download complete.") # Bucket for build resources. Necessary for fuzzer selection. logs.log("Fetching Fuchsia build.") fuchsia_build_url = environment.get_value('FUCHSIA_BUILD_URL') if not fuchsia_build_url: raise errors.FuchsiaConfigError('Could not find path for remote' 'Fuchsia build bucket (FUCHSIA BUILD URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_build_url, fuchsia_resources_dir ] logs.log("Beginning Fuchsia build download.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia ' 'resources: ' + result.output) return fuchsia_resources_dir
def save_corpus(self, source_dir, destination_dir): """Saves syzkaller to folder so it is backed up to the cloud. Args: source_dir: Folder where syzkaller corpus is. destination_dir: Folder where the corpus is synced with the cloud. """ source_file = os.path.join(source_dir, CORPUS_DB_FILENAME) shell.create_directory(destination_dir) destination_file = os.path.join(destination_dir, self._get_device_corpus_db_filename()) if os.path.isfile(source_file) and ( not os.path.exists(destination_file) or (os.path.getsize(source_file) > os.path.getsize(destination_file))): shutil.copy(source_file, destination_file)
def _setup_build_directories(base_build_dir): """Set up build directories for a job.""" # Create the root build directory for this job. shell.create_directory(base_build_dir, create_intermediates=True) custom_binary_directory = os.path.join(base_build_dir, 'custom') revision_build_directory = os.path.join(base_build_dir, 'revisions') sym_build_directory = os.path.join(base_build_dir, 'symbolized') sym_debug_build_directory = os.path.join(sym_build_directory, 'debug') sym_release_build_directory = os.path.join(sym_build_directory, 'release') build_directories = [ custom_binary_directory, revision_build_directory, sym_build_directory, sym_debug_build_directory, sym_release_build_directory ] for build_directory in build_directories: shell.create_directory(build_directory)
def setup_user_profile_directory_if_needed(user_profile_directory): """Set user profile directory if it does not exist.""" if os.path.exists(user_profile_directory): # User profile directory already exists. Bail out. return shell.create_directory(user_profile_directory) # Create a file in user profile directory based on format: # filename;base64 encoded zlib compressed file contents. user_profile_file = environment.get_value('USER_PROFILE_FILE') if user_profile_file and ';' in user_profile_file: user_profile_filename, encoded_file_contents = ( user_profile_file.split(';', 1)) user_profile_file_contents = zlib.decompress( base64.b64decode(encoded_file_contents)) user_profile_file_path = os.path.join(user_profile_directory, user_profile_filename) utils.write_data_to_file(user_profile_file_contents, user_profile_file_path) # For Firefox, we need to install a special fuzzPriv extension that exposes # special functions to javascript, e.g. gc(), etc. app_name = environment.get_value('APP_NAME') if app_name.startswith('firefox'): # Create extensions directory. extensions_directory = os.path.join(user_profile_directory, 'extensions') shell.create_directory(extensions_directory) # Unpack the fuzzPriv extension. extension_archive = os.path.join(environment.get_resources_directory(), 'firefox', 'fuzzPriv-extension.zip') archive.unpack(extension_archive, extensions_directory) # Add this extension in the extensions configuration file. extension_config_file_path = os.path.join(user_profile_directory, 'extensions.ini') fuzz_extension_directory = os.path.join(extensions_directory, '*****@*****.**') extension_config_file_contents = ('[ExtensionDirs]\r\n' 'Extension0=%s\r\n' '\r\n' '[ThemeDirs]\r\n' % fuzz_extension_directory) utils.write_data_to_file(extension_config_file_contents, extension_config_file_path)
def mocked_create_merge_directory(_): """A mocked version of create_merge_directory that adds some interesting files to the merge corpus and initial corpus.""" merge_directory_path = libfuzzer.create_corpus_directory('merge-corpus') shell.create_directory( merge_directory_path, create_intermediates=True, recreate=True) # Write the minimal unit to the merge directory. minimal_unit_path = os.path.join(merge_directory_path, minimal_unit_hash) with open(minimal_unit_path, 'w+') as file_handle: file_handle.write(minimal_unit_contents) # Write the nonminimal unit to the corpus directory. nonminimal_unit_path = os.path.join(corpus_path, nonminimal_unit_hash) with open(nonminimal_unit_path, 'w+') as file_handle: file_handle.write(nonminimal_unit_contents) return merge_directory_path
def test_file_exists(self): """Tests file_exists.""" utils.write_data_to_file('a', os.path.join(self.local_temp_dir, 'a')) shell.create_directory(os.path.join(self.local_temp_dir, 'b')) utils.write_data_to_file('c', os.path.join(self.local_temp_dir, 'b', 'c')) adb.copy_local_directory_to_remote(self.local_temp_dir, self.device_temp_dir) existent_file_path_remote = os.path.join(self.device_temp_dir, 'a') existent_directory_path_remote = os.path.join(self.device_temp_dir, 'b') non_existent_file_path_remote = os.path.join(self.device_temp_dir, 'd') non_existent_directory_path_remote = os.path.join(self.device_temp_dir, 'e') self.assertTrue(adb.file_exists(existent_file_path_remote)) self.assertFalse(adb.file_exists(existent_directory_path_remote)) self.assertFalse(adb.file_exists(non_existent_file_path_remote)) self.assertFalse(adb.file_exists(non_existent_directory_path_remote))
def initialize_resources_dir(): """Download Fuchsia QEMU resources from GCS bucket.""" # This module depends on multiprocessing, which is not available in # appengine, and since appengine *imports* this file (but does not run this # function!), we import it here. from google_cloud_utils import gsutil resources_dir = environment.get_value('RESOURCES_DIR') if not resources_dir: raise errors.FuchsiaConfigError('Could not find RESOURCES_DIR') fuchsia_resources_dir = os.path.join(resources_dir, 'fuchsia') shell.create_directory(fuchsia_resources_dir, create_intermediates=True, recreate=True) # Bucket for QEMU resources. fuchsia_resources_url = environment.get_value('FUCHSIA_BUILD_URL') if not fuchsia_resources_url: raise errors.FuchsiaConfigError( 'Could not find path for remote' 'Fuchsia resources bucket (FUCHSIA_BUILD_URL') gsutil_command_arguments = [ '-m', 'cp', '-r', fuchsia_resources_url, fuchsia_resources_dir ] logs.log("Fetching Fuchsia build.") result = gsutil.GSUtilRunner().run_gsutil(gsutil_command_arguments) if result.return_code or result.timed_out: raise errors.FuchsiaSdkError('Failed to download Fuchsia ' 'resources: ' + result.output) # Chmod the symbolizers so they can be used easily. symbolizer_path = os.path.join(fuchsia_resources_dir, 'build', 'zircon', 'prebuilt', 'downloads', 'symbolize', 'linux-x64', 'symbolize') llvm_symbolizer_path = os.path.join(fuchsia_resources_dir, 'build', 'buildtools', 'linux-x64', 'clang', 'bin', 'llvm-symbolizer') os.chmod(symbolizer_path, 0o111) os.chmod(llvm_symbolizer_path, 0o111) logs.log("Fuchsia build download complete.") return fuchsia_resources_dir
def mocked_create_merge_directory(): """A mocked version of create_merge_directory that adds some interesting files to the merge corpus and initial corpus.""" merge_directory_path = launcher.get_merge_directory() shell.create_directory( merge_directory_path, create_intermediates=True, recreate=True) # Write the minimal unit to the merge directory. minimal_unit_path = os.path.join(merge_directory_path, minimal_unit_hash) with open(minimal_unit_path, 'w+') as file_handle: file_handle.write(minimal_unit_contents) # Write the nonminimal unit to the corpus directory. corpus_directory = os.getenv('FUZZ_CORPUS_DIR') nonminimal_unit_path = os.path.join(corpus_directory, nonminimal_unit_hash) with open(nonminimal_unit_path, 'w+') as file_handle: file_handle.write(nonminimal_unit_contents) return merge_directory_path
def request(url, body=None, method=POST_METHOD, force_reauthorization=False, configuration=None): """Make an HTTP request to the specified URL.""" if configuration: authorization = _get_authorization(force_reauthorization, configuration) headers = { "User-Agent": "clusterfuzz-reproduce", "Authorization": authorization, } else: headers = {} http = httplib2.Http() request_body = json_utils.dumps(body) if body is not None else "" response, content = http.request(url, method=method, headers=headers, body=request_body) # If the server returns 401 we may need to reauthenticate. Try the request # a second time if this happens. if response.status == 401 and not force_reauthorization: return request( url, body, method=method, force_reauthorization=True, configuration=configuration, ) if AUTHORIZATION_HEADER in response: shell.create_directory(os.path.dirname(AUTHORIZATION_CACHE_FILE), create_intermediates=True) utils.write_data_to_file(response[AUTHORIZATION_HEADER], AUTHORIZATION_CACHE_FILE) return response, content
def rsync_to_disk(self, directory, timeout=CORPUS_FILES_SYNC_TIMEOUT, delete=True): """Run gsutil to download corpus files from GCS. Args: directory: Path to directory to sync to. timeout: Timeout for gsutil. delete: Whether or not to delete files on disk that don't exist locally. Returns: A bool indicating whether or not the command succeeded. """ shell.create_directory(directory, create_intermediates=True) corpus_gcs_url = self.get_gcs_url() result = self._gsutil_runner.rsync(corpus_gcs_url, directory, timeout, delete) # Allow a small number of files to fail to be synced. return _handle_rsync_result(result, max_errors=MAX_SYNC_ERRORS)
def create_directories(self): """Creates directories needed to use mutator plugins.""" # TODO(320): Change mutator plugin downloads so that they don't need to be # deleted and redownloaded on each run of launcher.py. shell.create_directory(environment.get_value('MUTATOR_PLUGINS_DIR'), create_intermediates=True, recreate=True) shell.create_directory(_get_mutator_plugins_archives_dir(), create_intermediates=True, recreate=True) shell.create_directory(_get_mutator_plugins_unpacked_dir(), create_intermediates=True, recreate=True)
def unpack_crash_testcases(crash_testcases_directory): """Unpacks the old crash testcases in the provided directory.""" for testcase in ndb_utils.get_all_from_model(data_types.Testcase): testcase_id = testcase.key.id() # 1. If we have already stored the testcase, then just skip. if testcase_id in STORED_TESTCASES_LIST: continue # 2. Make sure that it is a unique crash testcase. Ignore duplicates, # uploaded repros. if testcase.status != 'Processed': continue # 3. Check if the testcase is fixed. If not, skip. if testcase.open: continue # 4. Check if the testcase has a minimized repro. If not, skip. if not testcase.minimized_keys or testcase.minimized_keys == 'NA': continue # 5. Only use testcases that have bugs associated with them. if not testcase.bug_information: continue # 6. Existing IPC testcases are un-interesting and unused in furthur # mutations. Due to size bloat, ignoring these for now. if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION): continue # 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). if archive.get_archive_type(testcase.absolute_path): continue # 8. Skip in-process fuzzer testcases, since these are only applicable to # fuzz targets and don't run with blackbox binaries. if testcase.fuzzer_name and testcase.fuzzer_name in [ 'afl', 'libFuzzer' ]: continue # Un-pack testcase. try: _, input_directory, _ = setup.unpack_testcase(testcase) except Exception: logs.log_error('Failed to unpack testcase %d.' % testcase.key.id()) continue # Move this to our crash testcases directory. crash_testcase_directory = os.path.join(crash_testcases_directory, str(testcase_id)) shell.move(input_directory, crash_testcase_directory) # Re-create input directory for unpacking testcase in next iteration. shell.create_directory(input_directory) STORED_TESTCASES_LIST.append(testcase_id) # Remove testcase directories that exceed the max size limit. for directory_name in os.listdir(crash_testcases_directory): directory_path = os.path.join(crash_testcases_directory, directory_name) if not os.path.isdir(directory_path): continue if shell.get_directory_size( directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE: continue shell.remove_directory(directory_path) # Rename all fuzzed testcase files as regular files. for root, _, files in os.walk(crash_testcases_directory): for filename in files: if not filename.startswith(testcase_manager.FUZZ_PREFIX): continue file_path = os.path.join(root, filename) stripped_file_name = os.path.basename( file_path)[len(testcase_manager.FUZZ_PREFIX):] stripped_file_path = os.path.join(os.path.dirname(file_path), stripped_file_name) try: os.rename(file_path, stripped_file_path) except: raise Exception('Failed to rename testcase %s.' % file_path) # Remove empty files and dirs to avoid the case where a fuzzer randomly # chooses an empty dir/file and generates zero testcases. shell.remove_empty_files(crash_testcases_directory) shell.remove_empty_directories(crash_testcases_directory)
def main(): """Main sync routine.""" tests_archive_bucket = environment.get_value('TESTS_ARCHIVE_BUCKET') tests_archive_name = environment.get_value('TESTS_ARCHIVE_NAME') tests_directory = environment.get_value('TESTS_DIR') sync_interval = environment.get_value('SYNC_INTERVAL') # in seconds. shell.create_directory(tests_directory) # Sync old crash tests. logs.log('Syncing old crash tests.') crash_testcases_directory = os.path.join(tests_directory, 'CrashTests') shell.create_directory(crash_testcases_directory) unpack_crash_testcases(crash_testcases_directory) # Sync web tests. logs.log('Syncing web tests.') src_directory = os.path.join(tests_directory, 'src') gclient_file_path = os.path.join(tests_directory, '.gclient') if not os.path.exists(gclient_file_path): subprocess.check_call( ['fetch', '--no-history', 'chromium', '--nosvn=True'], cwd=tests_directory) if os.path.exists(src_directory): subprocess.check_call(['gclient', 'revert'], cwd=src_directory) subprocess.check_call(['git', 'pull'], cwd=src_directory) subprocess.check_call(['gclient', 'sync'], cwd=src_directory) else: raise Exception('Unable to checkout web tests.') clone_git_repository(tests_directory, 'v8', 'https://chromium.googlesource.com/v8/v8') clone_git_repository(tests_directory, 'ChakraCore', 'https://github.com/Microsoft/ChakraCore.git') clone_git_repository(tests_directory, 'gecko-dev', 'https://github.com/mozilla/gecko-dev.git') clone_git_repository(tests_directory, 'webgl-conformance-tests', 'https://github.com/KhronosGroup/WebGL.git') checkout_svn_repository( tests_directory, 'WebKit/LayoutTests', 'http://svn.webkit.org/repository/webkit/trunk/LayoutTests') checkout_svn_repository( tests_directory, 'WebKit/JSTests/stress', 'http://svn.webkit.org/repository/webkit/trunk/JSTests/stress') checkout_svn_repository( tests_directory, 'WebKit/JSTests/es6', 'http://svn.webkit.org/repository/webkit/trunk/JSTests/es6') create_gecko_tests_directory(tests_directory, 'gecko-dev', 'gecko-tests') # Upload tests archive to google cloud storage. logs.log('Uploading tests archive to cloud.') tests_archive_local = os.path.join(tests_directory, tests_archive_name) tests_archive_remote = 'gs://{bucket_name}/{archive_name}'.format( bucket_name=tests_archive_bucket, archive_name=tests_archive_name) shell.remove_file(tests_archive_local) create_symbolic_link(tests_directory, 'gecko-dev/js/src/tests', 'spidermonkey') create_symbolic_link(tests_directory, 'ChakraCore/test', 'chakra') # FIXME: Find a way to rename LayoutTests to web_tests without breaking # compatability with older testcases. create_symbolic_link(tests_directory, 'src/third_party/blink/web_tests', 'LayoutTests') subprocess.check_call([ 'zip', '-r', tests_archive_local, 'CrashTests', 'LayoutTests', 'WebKit', 'gecko-tests', 'v8/test/mjsunit', 'spidermonkey', 'chakra', 'webgl-conformance-tests', '-x', '*.cc', '-x', '*.cpp', '-x', '*.py', '-x', '*.txt', '-x', '*-expected.*', '-x', '*.git*', '-x', '*.svn*', ], cwd=tests_directory) subprocess.check_call( ['gsutil', 'cp', tests_archive_local, tests_archive_remote]) logs.log('Completed cycle, sleeping for %s seconds.' % sync_interval) time.sleep(sync_interval)
def update_data_bundle(fuzzer, data_bundle): """Updates a data bundle to the latest version.""" # This module can't be in the global imports due to appengine issues # with multiprocessing and psutil imports. from google_cloud_utils import gsutil # If we are using a data bundle on NFS, it is expected that our testcases # will usually be large enough that we would fill up our tmpfs directory # pretty quickly. So, change it to use an on-disk directory. if not data_bundle.is_local: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) data_bundle_directory = get_data_bundle_directory(fuzzer.name) if not data_bundle_directory: logs.log_error('Failed to setup data bundle %s.' % data_bundle.name) return False if not shell.create_directory(data_bundle_directory, create_intermediates=True): logs.log_error('Failed to create data bundle %s directory.' % data_bundle.name) return False # Check if data bundle is up to date. If yes, skip the update. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Data bundle was recently synced, skip.') return True # Fetch lock for this data bundle. if not _fetch_lock_for_data_bundle_update(data_bundle): logs.log_error('Failed to lock data bundle %s.' % data_bundle.name) return False # Re-check if another bot did the sync already. If yes, skip. if _is_data_bundle_up_to_date(data_bundle, data_bundle_directory): logs.log('Another bot finished the sync, skip.') _release_lock_for_data_bundle_update(data_bundle) return True time_before_sync_start = time.time() # No need to sync anything if this is a search index data bundle. In that # case, the fuzzer will generate testcases from a gcs bucket periodically. if not _is_search_index_data_bundle(data_bundle.name): bucket_url = data_handler.get_data_bundle_bucket_url(data_bundle.name) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import corpus_manager from bot.untrusted_runner import file_host worker_data_bundle_directory = file_host.rebase_to_worker_root( data_bundle_directory) file_host.create_directory(worker_data_bundle_directory, create_intermediates=True) result = corpus_manager.RemoteGSUtilRunner().rsync( bucket_url, worker_data_bundle_directory, delete=False) else: result = gsutil.GSUtilRunner().rsync(bucket_url, data_bundle_directory, delete=False) if result.return_code != 0: logs.log_error('Failed to sync data bundle %s: %s.' % (data_bundle.name, result.output)) _release_lock_for_data_bundle_update(data_bundle) return False # Update the testcase list file. testcase_manager.create_testcase_list_file(data_bundle_directory) # Write last synced time in the sync file. sync_file_path = _get_data_bundle_sync_file_path(data_bundle_directory) utils.write_data_to_file(time_before_sync_start, sync_file_path) if environment.is_trusted_host() and data_bundle.sync_to_worker: from bot.untrusted_runner import file_host worker_sync_file_path = file_host.rebase_to_worker_root(sync_file_path) file_host.copy_file_to_worker(sync_file_path, worker_sync_file_path) # Release acquired lock. _release_lock_for_data_bundle_update(data_bundle) return True
def copy_remote_file_to_local(remote_file_path, local_file_path): """Copies device file to a local file.""" shell.create_directory( os.path.dirname(local_file_path), create_intermediates=True) run_command(['pull', remote_file_path, local_file_path])
def store_file_in_cache(file_path, cached_files_per_directory_limit=True, force_update=False): """Get file from nfs cache if available.""" if not os.path.exists(file_path): logs.log_error( 'Local file %s does not exist, nothing to store in cache.' % file_path) return if os.path.getsize(file_path) > CACHE_SIZE_LIMIT: logs.log('File %s is too large to store in cache, skipping.' % file_path) return nfs_root = environment.get_value('NFS_ROOT') if not nfs_root: # No NFS, nothing to store in cache. return # If NFS server is not available due to heavy load, skip storage operation # altogether as we would fail to store file. if not os.path.exists(os.path.join(nfs_root, '.')): # Use . to iterate mount. logs.log_warn('Cache %s not available.' % nfs_root) return cache_file_path = get_cache_file_path(file_path) cache_directory = os.path.dirname(cache_file_path) filename = os.path.basename(file_path) if not os.path.exists(cache_directory): if not shell.create_directory(cache_directory, create_intermediates=True): logs.log_error('Failed to create cache directory %s.' % cache_directory) return # Check if the file already exists in cache. if file_exists_in_cache(cache_file_path): if not force_update: return # If we are forcing update, we need to remove current cached file and its # metadata. remove_cache_file_and_metadata(cache_file_path) # Delete old cached files beyond our maximum storage limit. if cached_files_per_directory_limit: # Get a list of cached files. cached_files_list = [] for cached_filename in os.listdir(cache_directory): if cached_filename.endswith(CACHE_METADATA_FILE_EXTENSION): continue cached_file_path = os.path.join(cache_directory, cached_filename) cached_files_list.append(cached_file_path) mtime = lambda f: os.stat(f).st_mtime last_used_cached_files_list = list( sorted(cached_files_list, key=mtime, reverse=True)) for cached_file_path in ( last_used_cached_files_list[MAX_CACHED_FILES_PER_DIRECTORY - 1:]): remove_cache_file_and_metadata(cached_file_path) # Start storing the actual file in cache now. logs.log('Started storing file %s into cache.' % filename) # Fetch lock to store this file. Try only once since if any other bot has # started to store it, we don't need to do it ourselves. Just bail out. lock_name = 'store:cache_file:%s' % utils.string_hash(cache_file_path) if not locks.acquire_lock( lock_name, max_hold_seconds=CACHE_LOCK_TIMEOUT, retries=1, by_zone=True): logs.log_warn( 'Unable to fetch lock to update cache file %s, skipping.' % filename) return # Check if another bot already updated it. if file_exists_in_cache(cache_file_path): locks.release_lock(lock_name, by_zone=True) return shell.copy_file(file_path, cache_file_path) write_cache_file_metadata(cache_file_path, file_path) time.sleep(CACHE_COPY_WAIT_TIME) error_occurred = not file_exists_in_cache(cache_file_path) locks.release_lock(lock_name, by_zone=True) if error_occurred: logs.log_error('Failed to store file %s into cache.' % filename) else: logs.log('Completed storing file %s into cache.' % filename)