def test_rsync(self): """Tests that rsync works as intended.""" with mock.patch( 'common.gsutil.gsutil_command') as mocked_gsutil_command: gsutil.rsync(self.SRC, self.DST) mocked_gsutil_command.assert_called_with( ['rsync', '-d', '-r', '/src', 'gs://dst'])
def test_options(self): """Tests that rsync works as intended when supplied a gsutil_options argument.""" flag = '-flag' with mock.patch( 'common.gsutil.gsutil_command') as mocked_gsutil_command: gsutil.rsync(self.SRC, self.DST, options=[flag]) assert flag in mocked_gsutil_command.call_args_list[0][0][0]
def test_no_flag(self, kwarg_for_rsync, flag): """Tests that rsync works as intended when caller specifies not to use specific flags.""" kwargs_for_rsync = {} kwargs_for_rsync[kwarg_for_rsync] = False with mock.patch( 'common.gsutil.gsutil_command') as mocked_gsutil_command: gsutil.rsync(self.SRC, self.DST, **kwargs_for_rsync) assert flag not in mocked_gsutil_command.call_args_list[0][0][0]
def save_results(self): """Save the results directory to GCS.""" if not self.gcs_sync_dir: return # Copy results directory before rsyncing it so that we don't get an # exception from uploading a file that changes in size. Files can change # in size because the log file containing the fuzzer's output is in this # directory and can be written to by the fuzzer at any time. results_copy = filesystem.make_dir_copy(self.results_dir) gsutil.rsync(results_copy, posixpath.join(self.gcs_sync_dir, self.results_dir))
def output_report(web_bucket): """Generate the HTML report and write it to |web_bucket|.""" experiment_name = experiment_utils.get_experiment_name() reports_dir = get_reports_dir() try: logger.debug('Generating report.') filesystem.recreate_directory(reports_dir) generate_report.generate_report([experiment_name], str(reports_dir)) gsutil.rsync(str(reports_dir), web_bucket, gsutil_options=[ '-h', 'Cache-Control:public,max-age=0,no-transform' ], parallel=False) logger.debug('Done generating report.') except Exception: # pylint: disable=broad-except logger.error('Error generating HTML report.')
def copy_resources_to_bucket(config_dir: str, config: Dict): """Copy resources the dispatcher will need for the experiment to the cloud_experiment_bucket.""" cloud_experiment_path = os.path.join(config['cloud_experiment_bucket'], config['experiment']) base_destination = os.path.join(cloud_experiment_path, 'input') # Send the local source repository to the cloud for use by dispatcher. # Local changes to any file will propagate. # Filter out unnecessary directories. options = [ '-x', ('^\\.git/|^\\.pytype/|^\\.venv/|^.*\\.pyc$|^__pycache__/' '|.*~$|\\.pytest_cache/|.*/test_data/|^third_party/oss-fuzz/out/' '|^docs/') ] destination = os.path.join(base_destination, 'src') gsutil.rsync(utils.ROOT_DIR, destination, options=options) # Send config files. destination = os.path.join(base_destination, 'config') gsutil.rsync(config_dir, destination)
def copy_resources_to_bucket(config_dir: str, config: Dict): """Copy resources the dispatcher will need for the experiment to the cloud_experiment_bucket.""" def filter_file(tar_info): """Filter out unnecessary directories.""" if FILTER_SOURCE_REGEX.match(tar_info.name): return None return tar_info cloud_experiment_path = os.path.join(config['cloud_experiment_bucket'], config['experiment']) base_destination = os.path.join(cloud_experiment_path, 'input') # Send the local source repository to the cloud for use by dispatcher. # Local changes to any file will propagate. source_archive = 'src.tar.gz' with tarfile.open(source_archive, 'w:gz') as tar: tar.add(utils.ROOT_DIR, arcname='', recursive=True, filter=filter_file) gsutil.cp(source_archive, base_destination + '/', parallel=True) os.remove(source_archive) # Send config files. destination = os.path.join(base_destination, 'config') gsutil.rsync(config_dir, destination, parallel=True)
def measure_all_trials(experiment: str, max_total_time: int, pool, q) -> bool: # pylint: disable=invalid-name """Get coverage data (with coverage runs) for all active trials. Note that this should not be called unless multiprocessing.set_start_method('spawn') was called first. Otherwise it will use fork which breaks logging.""" logger.info('Measuring all trials.') experiment_folders_dir = get_experiment_folders_dir() if not remote_dir_exists(experiment_folders_dir): return True try: gsutil.rsync(exp_path.gcs(experiment_folders_dir), str(experiment_folders_dir)) except subprocess.CalledProcessError: logger.error('Rsyncing experiment folders failed.') return True max_cycle = _time_to_cycle(max_total_time) unmeasured_snapshots = get_unmeasured_snapshots(experiment, max_cycle) if not unmeasured_snapshots: return False measure_trial_coverage_args = [ (unmeasured_snapshot, max_cycle, q) for unmeasured_snapshot in unmeasured_snapshots ] result = pool.starmap_async(measure_trial_coverage, measure_trial_coverage_args) # Poll the queue for snapshots and save them in batches until the pool is # done processing each unmeasured snapshot. Then save any remaining # snapshots. snapshots = [] snapshots_measured = False def save_snapshots(): """Saves measured snapshots if there were any, resets |snapshots| to an empty list and records the fact that snapshots have been measured.""" if not snapshots: return db_utils.bulk_save(snapshots) snapshots.clear() nonlocal snapshots_measured snapshots_measured = True while True: try: snapshot = q.get(timeout=SNAPSHOT_QUEUE_GET_TIMEOUT) snapshots.append(snapshot) except queue.Empty: if result.ready(): # If "ready" that means pool has finished calling on each # unmeasured_snapshot. Since it is finished and the queue is # empty, we can stop checking the queue for more snapshots. break if len(snapshots) >= SNAPSHOTS_BATCH_SAVE_SIZE * .75: # Save a smaller batch size if we can make an educated guess # that we will have to wait for the next snapshot. save_snapshots() continue if len(snapshots) >= SNAPSHOTS_BATCH_SAVE_SIZE and not result.ready(): save_snapshots() # If we have any snapshots left save them now. save_snapshots() return snapshots_measured