def download_artifact_if_needed(build_id, artifact_directory, artifact_archive_path, targets_with_type_and_san, artifact_file_name, output_filename_override, build_params, build_params_check_path): """Downloads artifact to actifacts_archive_path if needed""" # Check if we already have the symbols in cache. cached_build_params = utils.read_data_from_file(build_params_check_path, eval_data=True) if cached_build_params and cached_build_params == build_params: # No work to do, same system symbols already in cache. return # Delete existing symbols directory first. shell.remove_directory(artifact_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( artifact_archive_path, update_modification_time_on_access=False) if not found_in_cache: for target_with_type_and_san in targets_with_type_and_san: # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name, artifact_directory, output_filename_override) if os.path.exists(artifact_archive_path): break
def setUp(self): self.tmp_dir = tempfile.mkdtemp() os.environ['BOT_TMPDIR'] = os.path.join(self.tmp_dir, 'bot_tmpdir') test_helpers.patch(self, [ 'clusterfuzz._internal.datastore.data_handler.' 'get_data_bundle_bucket_name', 'clusterfuzz._internal.system.environment.' 'set_environment_parameters_from_file', ]) test_helpers.patch_environ(self) # Our tests write data/logs into subdirs of ROOT_DIR. Pivot the ROOT_DIR to # a temporary one. new_root = _create_test_root() os.environ['ROOT_DIR'] = new_root self.saved_cwd = os.getcwd() os.chdir(new_root) environment.set_bot_environment() fuzz_inputs = os.environ['FUZZ_INPUTS'] shell.remove_directory(fuzz_inputs, recreate=True) worker_fuzz_inputs = file_host.rebase_to_worker_root(fuzz_inputs) shell.remove_directory(worker_fuzz_inputs, recreate=True) environment.set_value('GSUTIL_PATH', os.path.dirname(_which('gsutil'))) test_utils.setup_pubsub('test-clusterfuzz') test_utils.create_pubsub_topic(pubsub.PubSubClient(), 'test-clusterfuzz', 'jobs-project-linux')
def remove_binding(self, binding): """Overriden version of remove_binding that ensures the bound directory is removed. This is necessary because unlike in regular minijails, we do not delete the entire chroot directory.""" abs_path = os.path.join(self._chroot_dir, binding.dest_path[1:]) shell.remove_directory(abs_path) super(ChromeOSChroot, self).remove_binding(binding)
def clean_temp_directories(): """Clean temporary directories.""" for temp_directory in TEMP_DIRECTORIES: temp_directory_full_path = os.path.abspath( os.path.expandvars(temp_directory)) shell.remove_directory(temp_directory_full_path, recreate=True, ignore_errors=True)
def test_unpack_file_with_cwd_prefix(self): """Test unpack with trusted=False passes with file having './' prefix.""" tgz_path = os.path.join(TESTDATA_PATH, 'cwd-prefix.tgz') output_directory = tempfile.mkdtemp(prefix='cwd-prefix') archive.unpack(tgz_path, output_directory, trusted=False) test_file_path = os.path.join(output_directory, 'test') self.assertTrue(os.path.exists(test_file_path)) self.assertEqual(open(test_file_path).read(), 'abc\n') shell.remove_directory(output_directory)
def update_tests_if_needed(): """Updates layout tests every day.""" data_directory = environment.get_value('FUZZ_DATA') error_occured = False expected_task_duration = 60 * 60 # 1 hour. retry_limit = environment.get_value('FAIL_RETRIES') temp_archive = os.path.join(data_directory, 'temp.zip') tests_url = environment.get_value('WEB_TESTS_URL') # Check if we have a valid tests url. if not tests_url: return # Layout test updates are usually disabled to speedup local testing. if environment.get_value('LOCAL_DEVELOPMENT'): return # |UPDATE_WEB_TESTS| env variable can be used to control our update behavior. if not environment.get_value('UPDATE_WEB_TESTS'): return last_modified_time = persistent_cache.get_value( TESTS_LAST_UPDATE_KEY, constructor=datetime.datetime.utcfromtimestamp) if (last_modified_time is not None and not dates.time_has_expired(last_modified_time, days=TESTS_UPDATE_INTERVAL_DAYS)): return logs.log('Updating layout tests.') tasks.track_task_start(tasks.Task('update_tests', '', ''), expected_task_duration) # Download and unpack the tests archive. for _ in range(retry_limit): try: shell.remove_directory(data_directory, recreate=True) storage.copy_file_from(tests_url, temp_archive) archive.unpack(temp_archive, data_directory, trusted=True) shell.remove_file(temp_archive) error_occured = False break except: logs.log_error( 'Could not retrieve and unpack layout tests archive. Retrying.' ) error_occured = True if not error_occured: persistent_cache.set_value(TESTS_LAST_UPDATE_KEY, time.time(), persist_across_reboots=True) tasks.track_task_end()
def _download_testcase(testcase_id, testcase, configuration): """Download the test case and return its path.""" print('Downloading testcase...') testcase_download_url = '{url}?id={id}'.format( url=configuration.get('testcase_download_url'), id=testcase_id) response, content = http_utils.request(testcase_download_url, method=http_utils.GET_METHOD, configuration=configuration) if response.status != 200: raise errors.ReproduceToolUnrecoverableError( 'Unable to download test case.') bot_absolute_filename = response[FILENAME_RESPONSE_HEADER] # Store the test case in the config directory for debuggability. testcase_directory = os.path.join(CONFIG_DIRECTORY, 'current-testcase') shell.remove_directory(testcase_directory, recreate=True) environment.set_value('FUZZ_INPUTS', testcase_directory) testcase_path = os.path.join(testcase_directory, os.path.basename(bot_absolute_filename)) utils.write_data_to_file(content, testcase_path) # Unpack the test case if it's archived. # TODO(mbarbella): Rewrite setup.unpack_testcase and share this code. if testcase.minimized_keys and testcase.minimized_keys != 'NA': mask = data_types.ArchiveStatus.MINIMIZED else: mask = data_types.ArchiveStatus.FUZZED if testcase.archive_state & mask: archive.unpack(testcase_path, testcase_directory) file_list = archive.get_file_list(testcase_path) testcase_path = None for file_name in file_list: if os.path.basename(file_name) == os.path.basename( testcase.absolute_path): testcase_path = os.path.join(testcase_directory, file_name) break if not testcase_path: raise errors.ReproduceToolUnrecoverableError( 'Test case file was not found in archive.\n' 'Original filename: {absolute_path}.\n' 'Archive contents: {file_list}'.format( absolute_path=testcase.absolute_path, file_list=file_list)) return testcase_path
def test_remove_shutil_success(self): """Test remove with shutil.""" self.mock.exists.side_effect = [True, True, False] self.assertTrue(shell.remove_directory('dir')) self.mock.system.assert_has_calls( [mock.call('rm -rf "dir" > /dev/null 2>&1')]) self.mock.rmtree.assert_has_calls([mock.call('dir', onerror=mock.ANY)])
def download_artifact_if_needed( build_id, artifact_directory, artifact_archive_path, targets_with_type_and_san, artifact_file_name, output_filename_override): """Downloads artifact to actifacts_archive_path if needed""" # Delete existing symbols directory first. shell.remove_directory(artifact_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( artifact_archive_path, update_modification_time_on_access=False) if not found_in_cache: for target_with_type_and_san in targets_with_type_and_san: # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name, artifact_directory, output_filename_override) if os.path.exists(artifact_archive_path): break
def test_remove_shutil_failure_ignore_errors(self): self.mock.exists.side_effect = [True, True, True] self.assertFalse(shell.remove_directory('dir', ignore_errors=True)) self.mock.log_warn.assert_has_calls( [mock.call('Failed to clear directory dir.')]) self.assertEqual(0, self.mock.log_error.call_count) self.mock.system.assert_has_calls( [mock.call('rm -rf "dir" > /dev/null 2>&1')]) self.mock.rmtree.assert_has_calls([mock.call('dir', onerror=mock.ANY)])
def _clear_old_data_bundles_if_needed(): """Clear old data bundles so as to keep the disk cache restricted to |_DATA_BUNDLE_CACHE_COUNT| data bundles and prevent potential out-of-disk spaces.""" data_bundles_directory = environment.get_value('DATA_BUNDLES_DIR') dirs = [] for filename in os.listdir(data_bundles_directory): file_path = os.path.join(data_bundles_directory, filename) if not os.path.isdir(file_path): continue dirs.append(file_path) dirs_to_remove = sorted(dirs, key=os.path.getmtime, reverse=True)[_DATA_BUNDLE_CACHE_COUNT:] for dir_to_remove in dirs_to_remove: logs.log( 'Removing data bundle directory to keep disk cache small: %s' % dir_to_remove) shell.remove_directory(dir_to_remove)
def download_latest_build(build_info, image_regexes, image_directory): """Download the latest build artifact for the given branch and target.""" # Check if our local build matches the latest build. If not, we will # download it. build_id = build_info['bid'] target = build_info['target'] last_build_info = persistent_cache.get_value(constants.LAST_FLASH_BUILD_KEY) if last_build_info and last_build_info['bid'] == build_id: return # Clean up the images directory first. shell.remove_directory(image_directory, recreate=True) for image_regex in image_regexes: image_file_path = fetch_artifact.get(build_id, target, image_regex, image_directory) if not image_file_path: logs.log_error('Failed to download artifact %s for ' 'branch %s and target %s.' % (image_file_path, build_info['branch'], target)) return if image_file_path.endswith('.zip') or image_file_path.endswith('.tar.gz'): archive.unpack(image_file_path, image_directory)
def prepare_model_directory(fuzzer_name): """Prepare model directory, and return model path. Args: fuzzer_name: Name of the fuzzer to which this model belongs. Returns: Model path. For example, if `/tmp/model` is the directory containing model files(e.g. rnn.index), the path should be '/tmp/model/rnn'. """ # Get temporary directory. temp_directory = environment.get_value('BOT_TMPDIR') # Create model directory. model_directory = os.path.join(temp_directory, fuzzer_name) shell.remove_directory(model_directory, recreate=True) if not download_model_from_gcs(model_directory, fuzzer_name): return None # Got the model. Return model path. return os.path.join(model_directory, constants.RNN_MODEL_NAME)
def test_remove_shutil_onerror(self): """Test shutil invoking onerror.""" self.mock.exists.side_effect = [True, True, False] self.assertTrue(shell.remove_directory('dir')) self.mock.system.assert_has_calls( [mock.call('rm -rf "dir" > /dev/null 2>&1')]) self.mock.rmtree.assert_has_calls([mock.call('dir', onerror=mock.ANY)]) onerror = self.mock.rmtree.call_args[1]['onerror'] fake_fn = mock.MagicMock() fake_fn.side_effect = OSError() onerror(fake_fn, 'dir/child', ImportError()) self.mock.chmod.assert_has_calls([mock.call('dir/child', 0o750)]) fake_fn.assert_has_calls([mock.call('dir/child')])
def _test_remove_os_specific(self, platform, recreate, raise_makedirs_error): """Helper for testing removing dir with os-specific command.""" self.mock.platform.return_value = platform self.mock.exists.side_effect = [True, False, False] if raise_makedirs_error: self.mock.makedirs.side_effect = OSError() result = shell.remove_directory('dir', recreate=recreate) if recreate: self.assertEqual(not raise_makedirs_error, result) else: self.assertTrue(result) self.mock.rmtree.assert_has_calls([]) if recreate: self.mock.makedirs.assert_has_calls([mock.call('dir')]) else: self.mock.makedirs.assert_has_calls([])
def do_corpus_pruning(context, last_execution_failed, revision): """Run corpus pruning.""" # Set |FUZZ_TARGET| environment variable to help with unarchiving only fuzz # target and its related files. environment.set_value('FUZZ_TARGET', context.fuzz_target.binary) if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import tasks_host return tasks_host.do_corpus_pruning(context, last_execution_failed, revision) if not build_manager.setup_build(revision=revision): raise CorpusPruningException('Failed to setup build.') build_directory = environment.get_value('BUILD_DIR') start_time = datetime.datetime.utcnow() runner = Runner(build_directory, context) pruner = CorpusPruner(runner) fuzzer_binary_name = os.path.basename(runner.target_path) # If our last execution failed, shrink to a randomized corpus of usable size # to prevent corpus from growing unbounded and recurring failures when trying # to minimize it. if last_execution_failed: for corpus_url in [ context.corpus.get_gcs_url(), context.quarantine_corpus.get_gcs_url() ]: _limit_corpus_size(corpus_url) # Get initial corpus to process from GCS. context.sync_to_disk() initial_corpus_size = shell.get_directory_file_count( context.initial_corpus_path) # Restore a small batch of quarantined units back to corpus. context.restore_quarantined_units() # Shrink to a minimized corpus using corpus merge. pruner_stats = pruner.run(context.initial_corpus_path, context.minimized_corpus_path, context.bad_units_path) # Sync minimized corpus back to GCS. context.sync_to_gcs() # Create corpus backup. # Temporarily copy the past crash regressions folder into the minimized corpus # so that corpus backup archive can have both. regressions_input_dir = os.path.join(context.initial_corpus_path, 'regressions') regressions_output_dir = os.path.join(context.minimized_corpus_path, 'regressions') if shell.get_directory_file_count(regressions_input_dir): shutil.copytree(regressions_input_dir, regressions_output_dir) backup_bucket = environment.get_value('BACKUP_BUCKET') corpus_backup_url = corpus_manager.backup_corpus( backup_bucket, context.corpus, context.minimized_corpus_path) shell.remove_directory(regressions_output_dir) minimized_corpus_size_units = shell.get_directory_file_count( context.minimized_corpus_path) minimized_corpus_size_bytes = shell.get_directory_size( context.minimized_corpus_path) logs.log('Corpus pruned from %d to %d units.' % (initial_corpus_size, minimized_corpus_size_units)) # Process bad units found during merge. # Mapping of crash state -> CorpusCrash crashes = {} pruner.process_bad_units(context.bad_units_path, context.quarantine_corpus_path, crashes) context.quarantine_corpus.rsync_from_disk(context.quarantine_corpus_path) # Store corpus stats into CoverageInformation entity. project_qualified_name = context.fuzz_target.project_qualified_name() today = datetime.datetime.utcnow().date() coverage_info = data_types.CoverageInformation( fuzzer=project_qualified_name, date=today) quarantine_corpus_size = shell.get_directory_file_count( context.quarantine_corpus_path) quarantine_corpus_dir_size = shell.get_directory_size( context.quarantine_corpus_path) # Save the minimize corpus size before cross pollination to put in BigQuery. pre_pollination_corpus_size = minimized_corpus_size_units # Populate coverage stats. coverage_info.corpus_size_units = minimized_corpus_size_units coverage_info.corpus_size_bytes = minimized_corpus_size_bytes coverage_info.quarantine_size_units = quarantine_corpus_size coverage_info.quarantine_size_bytes = quarantine_corpus_dir_size coverage_info.corpus_backup_location = corpus_backup_url coverage_info.corpus_location = context.corpus.get_gcs_url() coverage_info.quarantine_location = context.quarantine_corpus.get_gcs_url() # Calculate remaining time to use for shared corpus merging. time_remaining = _get_time_remaining(start_time) if time_remaining <= 0: logs.log_warn('Not enough time for shared corpus merging.') return None cross_pollinator = CrossPollinator(runner) pollinator_stats = cross_pollinator.run(time_remaining) context.sync_to_gcs() # Update corpus size stats. minimized_corpus_size_units = shell.get_directory_file_count( context.minimized_corpus_path) minimized_corpus_size_bytes = shell.get_directory_size( context.minimized_corpus_path) coverage_info.corpus_size_units = minimized_corpus_size_units coverage_info.corpus_size_bytes = minimized_corpus_size_bytes logs.log('Finished.') sources = ','.join([ fuzzer.fuzz_target.project_qualified_name() for fuzzer in context.cross_pollinate_fuzzers ]) cross_pollination_stats = None if pruner_stats and pollinator_stats: cross_pollination_stats = CrossPollinationStats( project_qualified_name, context.cross_pollination_method, sources, context.tag, initial_corpus_size, pre_pollination_corpus_size, pruner_stats['edge_coverage'], pollinator_stats['edge_coverage'], pruner_stats['feature_coverage'], pollinator_stats['feature_coverage']) return CorpusPruningResult(coverage_info=coverage_info, crashes=list(crashes.values()), fuzzer_binary_name=fuzzer_binary_name, revision=environment.get_value('APP_REVISION'), cross_pollination_stats=cross_pollination_stats)
def unpack_crash_testcases(crash_testcases_directory): """Unpacks the old crash testcases in the provided directory.""" for testcase in ndb_utils.get_all_from_model(data_types.Testcase): testcase_id = testcase.key.id() # 1. If we have already stored the testcase, then just skip. if testcase_id in STORED_TESTCASES_LIST: continue # 2. Make sure that it is a unique crash testcase. Ignore duplicates, # uploaded repros. if testcase.status != 'Processed': continue # 3. Check if the testcase is fixed. If not, skip. if testcase.open: continue # 4. Check if the testcase has a minimized repro. If not, skip. if not testcase.minimized_keys or testcase.minimized_keys == 'NA': continue # 5. Only use testcases that have bugs associated with them. if not testcase.bug_information: continue # 6. Existing IPC testcases are un-interesting and unused in further # mutations. Due to size bloat, ignoring these for now. if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION): continue # 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). if archive.get_archive_type(testcase.absolute_path): continue # 8. Skip in-process fuzzer testcases, since these are only applicable to # fuzz targets and don't run with blackbox binaries. if testcase.fuzzer_name and testcase.fuzzer_name in ['afl', 'libFuzzer']: continue # Un-pack testcase. try: _, input_directory, _ = setup.unpack_testcase(testcase) except Exception: logs.log_error('Failed to unpack testcase %d.' % testcase.key.id()) continue # Move this to our crash testcases directory. crash_testcase_directory = os.path.join(crash_testcases_directory, str(testcase_id)) shell.move(input_directory, crash_testcase_directory) # Re-create input directory for unpacking testcase in next iteration. shell.create_directory(input_directory) STORED_TESTCASES_LIST.append(testcase_id) # Remove testcase directories that exceed the max size limit. for directory_name in os.listdir(crash_testcases_directory): directory_path = os.path.join(crash_testcases_directory, directory_name) if not os.path.isdir(directory_path): continue if shell.get_directory_size(directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE: continue shell.remove_directory(directory_path) # Rename all fuzzed testcase files as regular files. for root, _, files in os.walk(crash_testcases_directory): for filename in files: if not filename.startswith(testcase_manager.FUZZ_PREFIX): continue file_path = os.path.join(root, filename) stripped_file_name = os.path.basename(file_path)[len( testcase_manager.FUZZ_PREFIX):] stripped_file_path = os.path.join( os.path.dirname(file_path), stripped_file_name) try: os.rename(file_path, stripped_file_path) except: raise Exception('Failed to rename testcase %s.' % file_path) # Remove empty files and dirs to avoid the case where a fuzzer randomly # chooses an empty dir/file and generates zero testcases. shell.remove_empty_files(crash_testcases_directory) shell.remove_empty_directories(crash_testcases_directory)
def tearDown(self): adb.remove_directory(self.device_temp_dir) shell.remove_directory(self.local_temp_dir)
def recreate_directory(directory_path): """Delete directory if exists, create empty directory. Throw an exception if either fails.""" if not shell.remove_directory(directory_path, recreate=True): raise Exception('Failed to recreate directory: ' + directory_path)
def update_fuzzer_and_data_bundles(fuzzer_name): """Update the fuzzer with a given name if necessary.""" fuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == fuzzer_name).get() if not fuzzer: logs.log_error('No fuzzer exists with name %s.' % fuzzer_name) raise errors.InvalidFuzzerError # Set some helper environment variables. fuzzer_directory = get_fuzzer_directory(fuzzer_name) environment.set_value('FUZZER_DIR', fuzzer_directory) environment.set_value('UNTRUSTED_CONTENT', fuzzer.untrusted_content) # If the fuzzer generates large testcases or a large number of small ones # that don't fit on tmpfs, then use the larger disk directory. if fuzzer.has_large_testcases: testcase_disk_directory = environment.get_value('FUZZ_INPUTS_DISK') environment.set_value('FUZZ_INPUTS', testcase_disk_directory) # Adjust the test timeout, if user has provided one. if fuzzer.timeout: environment.set_value('TEST_TIMEOUT', fuzzer.timeout) # Increase fuzz test timeout if the fuzzer timeout is higher than its # current value. fuzz_test_timeout = environment.get_value('FUZZ_TEST_TIMEOUT') if fuzz_test_timeout and fuzz_test_timeout < fuzzer.timeout: environment.set_value('FUZZ_TEST_TIMEOUT', fuzzer.timeout) # Adjust the max testcases if this fuzzer has specified a lower limit. max_testcases = environment.get_value('MAX_TESTCASES') if fuzzer.max_testcases and fuzzer.max_testcases < max_testcases: environment.set_value('MAX_TESTCASES', fuzzer.max_testcases) # Check for updates to this fuzzer. version_file = os.path.join(fuzzer_directory, '.%s_version' % fuzzer_name) if (not fuzzer.builtin and revisions.needs_update(version_file, fuzzer.revision)): logs.log('Fuzzer update was found, updating.') # Clear the old fuzzer directory if it exists. if not shell.remove_directory(fuzzer_directory, recreate=True): logs.log_error('Failed to clear fuzzer directory.') return None # Copy the archive to local disk and unpack it. archive_path = os.path.join(fuzzer_directory, fuzzer.filename) if not blobs.read_blob_to_disk(fuzzer.blobstore_key, archive_path): logs.log_error('Failed to copy fuzzer archive.') return None try: archive.unpack(archive_path, fuzzer_directory) except Exception: error_message = ( 'Failed to unpack fuzzer archive %s ' '(bad archive or unsupported format).') % fuzzer.filename logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None fuzzer_path = os.path.join(fuzzer_directory, fuzzer.executable_path) if not os.path.exists(fuzzer_path): error_message = ( 'Fuzzer executable %s not found. ' 'Check fuzzer configuration.') % fuzzer.executable_path logs.log_error(error_message) fuzzer_logs.upload_script_log('Fatal error: ' + error_message, fuzzer_name=fuzzer_name) return None # Make fuzzer executable. os.chmod(fuzzer_path, 0o750) # Cleanup unneeded archive. shell.remove_file(archive_path) # Save the current revision of this fuzzer in a file for later checks. revisions.write_revision_to_revision_file(version_file, fuzzer.revision) logs.log('Updated fuzzer to revision %d.' % fuzzer.revision) _clear_old_data_bundles_if_needed() # Setup data bundles associated with this fuzzer. data_bundles = ndb_utils.get_all_from_query( data_types.DataBundle.query( data_types.DataBundle.name == fuzzer.data_bundle_name)) for data_bundle in data_bundles: if not update_data_bundle(fuzzer, data_bundle): return None # Setup environment variable for launcher script path. if fuzzer.launcher_script: fuzzer_launcher_path = os.path.join(fuzzer_directory, fuzzer.launcher_script) environment.set_value('LAUNCHER_PATH', fuzzer_launcher_path) # For launcher script usecase, we need the entire fuzzer directory on the # worker. if environment.is_trusted_host(): from clusterfuzz._internal.bot.untrusted_runner import file_host worker_fuzzer_directory = file_host.rebase_to_worker_root( fuzzer_directory) file_host.copy_directory_to_worker(fuzzer_directory, worker_fuzzer_directory, replace=True) return fuzzer
def tearDown(self): shell.remove_directory(self.model_directory) shell.remove_directory(self.log_directory)
def remove_created_dirs(self, chroot_dir, minijail_created_dirs=None): if minijail_created_dirs is None: minijail_created_dirs = ['tmp', 'proc', 'dev'] for directory in minijail_created_dirs: shell.remove_directory(os.path.join(chroot_dir, directory))
def remove_directory(request, _): """Remove a directory.""" result = shell.remove_directory(request.path, request.recreate) return untrusted_runner_pb2.RemoveDirectoryResponse(result=result)
def _cleanup(): """Clean up after running the tool.""" temp_directory = environment.get_value('ROOT_DIR') assert 'tmp' in temp_directory shell.remove_directory(temp_directory)
def tearDown(self): shell.remove_directory(self.build_directory)
def execute_task(full_fuzzer_name, job_type): """Execute ML RNN training task. The task is training RNN model by default. If more models are developed, arguments can be modified to specify which model to use. Args: fuzzer_name: Name of fuzzer, e.g. libpng_read_fuzzer. job_type: Job type, e.g. libfuzzer_chrome_asan. """ del job_type # Sets up fuzzer binary build. fuzz_target = data_handler.get_fuzz_target(full_fuzzer_name) if not fuzz_target: logs.log_warn( f'Fuzzer not found: {full_fuzzer_name}, skip RNN training.') return fuzzer_name = fuzz_target.project_qualified_name() # Directory to place training files, such as logs, models, corpus. # Use |FUZZ_INPUTS_DISK| since it is not size constrained. temp_directory = environment.get_value('FUZZ_INPUTS_DISK') # Get corpus. corpus_directory = get_corpus_directory(temp_directory, fuzzer_name) shell.remove_directory(corpus_directory, recreate=True) logs.log('Downloading corpus backup for %s.' % fuzzer_name) if not ml_train_utils.get_corpus(corpus_directory, fuzzer_name): logs.log_error('Failed to download corpus backup for %s.' % fuzzer_name) return # Get the directory to save models. model_directory = get_model_files_directory(temp_directory, fuzzer_name) shell.remove_directory(model_directory, recreate=True) # Get the directory to save training logs. log_directory = get_model_log_directory(temp_directory, fuzzer_name) shell.remove_directory(log_directory, recreate=True) result = train_rnn(corpus_directory, model_directory, log_directory) # Training process exited abnormally but not caused by timeout, meaning # error occurred during execution. if result.return_code and not result.timed_out: if result.return_code == constants.ExitCode.CORPUS_TOO_SMALL: logs.log_warn( 'ML RNN training task for fuzzer %s aborted due to small corpus.' % fuzzer_name) else: logs.log_error( 'ML RNN training task for fuzzer %s failed with ExitCode = %d.' % (fuzzer_name, result.return_code), output=utils.decode_to_unicode(result.output)) return # Timing out may be caused by large training corpus, but intermediate models # are frequently saved and can be uploaded. if result.timed_out: logs.log_warn('ML RNN training task for %s timed out.' % fuzzer_name) upload_model_to_gcs(model_directory, fuzzer_name)
def cleanup(): """Clean up temporary metadata.""" shell.remove_directory(get_temp_dir())
def cleanup(self): """Cleanup state.""" for path in self._created_directories: shell.remove_directory(path)
def tearDown(self): shell.remove_directory(self.output_directory) shell.remove_directory(self.empty_directory)
def tearDown(self): shell.remove_directory(self.temp_directory)