def initialize_measurement_dirs(self): """Initialize directories that will be needed for measuring coverage.""" for directory in [self.corpus_dir, self.sancov_dir, self.crashes_dir]: filesystem.recreate_directory(directory) for directory in [self.report_dir, self.prev_corpus_dir]: pathlib.Path(directory).mkdir(exist_ok=True)
def output_report(experiment_config: dict, in_progress=False): """Generate the HTML report and write it to |web_bucket|.""" experiment_name = experiment_utils.get_experiment_name() web_filestore_path = posixpath.join(experiment_config['report_filestore'], experiment_name) reports_dir = get_reports_dir() # Don't merge with nonprivate experiments until the very end as doing it # while the experiment is in progress will produce unusable realtime # results. merge_with_nonprivate = (not in_progress and experiment_config.get( 'merge_with_nonprivate', False)) try: logger.debug('Generating report.') filesystem.recreate_directory(reports_dir) generate_report.generate_report( [experiment_name], str(reports_dir), in_progress=in_progress, merge_with_clobber_nonprivate=merge_with_nonprivate) filestore_utils.rsync(str(reports_dir), web_filestore_path, gsutil_options=[ '-h', 'Cache-Control:public,max-age=0,no-transform' ]) logger.debug('Done generating report.') except data_utils.EmptyDataError: logs.warning('No snapshot data.') except Exception: # pylint: disable=broad-except logger.error('Error generating HTML report.')
def set_up_experiment_config_file(config): """Set up the config file that will actually be used in the experiment (not the one given to run_experiment.py).""" filesystem.recreate_directory(CONFIG_DIR) experiment_config_filename = os.path.join(CONFIG_DIR, 'experiment.yaml') with open(experiment_config_filename, 'w') as experiment_config_file: yaml.dump(config, experiment_config_file, default_flow_style=False)
def initialize_measurement_dirs(self): """Initialize directories that will be needed for measuring coverage.""" for directory in [ self.corpus_dir, self.coverage_dir, self.crashes_dir ]: filesystem.recreate_directory(directory) filesystem.create_directory(self.report_dir)
def set_up_experiment_config_file(config): """Set up the config file that will actually be used in the experiment (not the one given to run_experiment.py).""" filesystem.recreate_directory(experiment_utils.CONFIG_DIR) experiment_config_filename = ( experiment_utils.get_internal_experiment_config_relative_path()) with open(experiment_config_filename, 'w') as experiment_config_file: yaml.dump(config, experiment_config_file, default_flow_style=False)
def initialize_directories(self): """Initialize directories needed for the trial.""" directories = [ self.corpus_dir, self.corpus_archives_dir, self.results_dir, ] for directory in directories: filesystem.recreate_directory(directory)
def build_all_measurers(benchmarks: List[str]) -> List[str]: """Build measurers for benchmarks.""" logger.info('Building measurers.') filesystem.recreate_directory(build_utils.get_coverage_binaries_dir()) benchmarks = [(benchmark, ) for benchmark in benchmarks] results = retry_build_loop(build_measurer, benchmarks) logger.info('Done building measurers.') # Return list of benchmarks (like the list we were passed as an argument) # instead of returning a list of tuples each containing a benchmark. return [result[0] for result in results]
def build_all_measurers(benchmarks: List[str]) -> List[str]: """Build measurers for each benchmark in |benchmarks| in parallel Returns a list of benchmarks built successfully.""" logger.info('Building measurers.') filesystem.recreate_directory(build_utils.get_coverage_binaries_dir()) build_measurer_args = [(benchmark, ) for benchmark in benchmarks] successful_calls = retry_build_loop(build_measurer, build_measurer_args) logger.info('Done building measurers.') # Return list of benchmarks (like the list we were passed as an argument) # instead of returning a list of tuples each containing a benchmark. return [successful_call[0] for successful_call in successful_calls]
def test_recreate_directory_existing(fs): """Tests that recreate_directory recreates a directory that already exists.""" new_directory = 'new-directory' os.mkdir(new_directory) new_file = os.path.join(new_directory, 'file') with open(new_file, 'w') as file_handle: file_handle.write('hi') filesystem.recreate_directory(new_directory) assert os.path.exists(new_directory) assert not os.path.exists(new_file)
def start_experiment(experiment_name: str, config_filename: str, benchmarks: List[str], fuzzers: List[str], fuzzer_configs: List[str]): """Start a fuzzer benchmarking experiment.""" validate_benchmarks(benchmarks) config = read_and_validate_experiment_config(config_filename) config['benchmarks'] = ','.join(benchmarks) validate_experiment_name(experiment_name) config['experiment'] = experiment_name config_dir = 'config' filesystem.recreate_directory(config_dir) experiment_config_filename = os.path.join(config_dir, 'experiment.yaml') with open(experiment_config_filename, 'w') as experiment_config_file: yaml.dump(config, experiment_config_file, default_flow_style=False) if not fuzzers and not fuzzer_configs: raise Exception('Need to provide either a list of fuzzers or ' 'a list of fuzzer configs.') fuzzer_config_dir = os.path.join(config_dir, 'fuzzer-configs') filesystem.recreate_directory(fuzzer_config_dir) for fuzzer_config in fuzzer_configs: if fuzzer_configs.count(fuzzer_config) > 1: raise Exception('Fuzzer config "%s" provided more than once.' % fuzzer_config) # Validate the fuzzer yaml attributes e.g. fuzzer, env, etc. validate_fuzzer_config(fuzzer_config) shutil.copy(fuzzer_config, fuzzer_config_dir) for fuzzer in fuzzers: if fuzzers.count(fuzzer) > 1: raise Exception('Fuzzer "%s" provided more than once.' % fuzzer) validate_fuzzer(fuzzer) fuzzer_config_file_path = os.path.join(fuzzer_config_dir, fuzzer) # Create a simple yaml with just the fuzzer attribute. with open(fuzzer_config_file_path, 'w') as file_handle: file_handle.write('fuzzer: ' + fuzzer) # Make sure we can connect to database. if 'POSTGRES_PASSWORD' not in os.environ: raise Exception('Must set POSTGRES_PASSWORD environment variable.') gcloud.set_default_project(config['cloud_project']) dispatcher = Dispatcher(config) if not os.getenv('MANUAL_EXPERIMENT'): dispatcher.create_async() copy_resources_to_bucket(config_dir, config) if not os.getenv('MANUAL_EXPERIMENT'): dispatcher.start()
def set_up_fuzzer_config_files(fuzzer_configs): """Write configurations specified by |fuzzer_configs| to yaml files that will be used to store configurations.""" if not fuzzer_configs: raise Exception('Need to provide either a list of fuzzers or ' 'a list of fuzzer configs.') fuzzer_config_dir = os.path.join(CONFIG_DIR, 'fuzzer-configs') filesystem.recreate_directory(fuzzer_config_dir) for fuzzer_config in fuzzer_configs: # Validate the fuzzer yaml attributes e.g. fuzzer, env, etc. validate_fuzzer_config(fuzzer_config) config_file_name = os.path.join(fuzzer_config_dir, get_full_fuzzer_name(fuzzer_config)) yaml_utils.write(config_file_name, fuzzer_config)
def output_report(experiment_config: dict, in_progress=False, coverage_report=False): """Generate the HTML report and write it to |web_bucket|.""" experiment_name = experiment_utils.get_experiment_name() reports_dir = get_reports_dir() core_fuzzers = set(get_core_fuzzers()) experiment_fuzzers = set(experiment_config['fuzzers']) fuzzers = experiment_fuzzers.union(core_fuzzers) # Calculate path to store report files in filestore. web_filestore_path = experiment_config['report_filestore'] if not fuzzers.issubset(core_fuzzers): # This means that we are running an experimental report with fuzzers # not in the core list. So, store these in |experimental| sub-directory. web_filestore_path = os.path.join(web_filestore_path, 'experimental') web_filestore_path = posixpath.join(web_filestore_path, experiment_name) # Don't merge with nonprivate experiments until the very end as doing it # while the experiment is in progress will produce unusable realtime # results. merge_with_nonprivate = (not in_progress and experiment_config.get( 'merge_with_nonprivate', False)) try: logger.debug('Generating report.') filesystem.recreate_directory(reports_dir) generate_report.generate_report( [experiment_name], str(reports_dir), report_name=experiment_name, fuzzers=fuzzers, in_progress=in_progress, merge_with_clobber_nonprivate=merge_with_nonprivate, coverage_report=coverage_report) filestore_utils.rsync( str(reports_dir), web_filestore_path, delete=False, # Don't remove existing coverage jsons. gsutil_options=[ '-h', 'Cache-Control:public,max-age=0,no-transform' ]) logger.debug('Done generating report.') except data_utils.EmptyDataError: logs.warning('No snapshot data.') except Exception: # pylint: disable=broad-except logger.error('Error generating HTML report.')
def output_report(web_bucket): """Generate the HTML report and write it to |web_bucket|.""" experiment_name = experiment_utils.get_experiment_name() reports_dir = get_reports_dir() try: logger.debug('Generating report.') filesystem.recreate_directory(reports_dir) generate_report.generate_report([experiment_name], str(reports_dir)) gsutil.rsync(str(reports_dir), web_bucket, gsutil_options=[ '-h', 'Cache-Control:public,max-age=0,no-transform' ], parallel=False) logger.debug('Done generating report.') except Exception: # pylint: disable=broad-except logger.error('Error generating HTML report.')
def output_report(web_bucket, in_progress=False): """Generate the HTML report and write it to |web_bucket|.""" experiment_name = experiment_utils.get_experiment_name() reports_dir = get_reports_dir() try: logger.debug('Generating report.') filesystem.recreate_directory(reports_dir) generate_report.generate_report([experiment_name], str(reports_dir), in_progress=in_progress) filestore_utils.rsync(str(reports_dir), web_bucket, gsutil_options=[ '-h', 'Cache-Control:public,max-age=0,no-transform' ]) logger.debug('Done generating report.') except data_utils.EmptyDataError: logs.warning('No snapshot data.') except Exception: # pylint: disable=broad-except logger.error('Error generating HTML report.')
def set_up_fuzzer_config_files(fuzzers, fuzzer_configs): """Set up config files for each fuzzer in |fuzzers| and each config file provided in |fuzzer_configs|.""" if not fuzzers and not fuzzer_configs: raise Exception('Need to provide either a list of fuzzers or ' 'a list of fuzzer configs.') fuzzer_config_dir = os.path.join(CONFIG_DIR, 'fuzzer-configs') filesystem.recreate_directory(fuzzer_config_dir) for fuzzer_config in fuzzer_configs: if fuzzer_configs.count(fuzzer_config) > 1: raise Exception('Fuzzer config "%s" provided more than once.' % fuzzer_config) # Validate the fuzzer yaml attributes e.g. fuzzer, env, etc. validate_fuzzer_config(fuzzer_config) shutil.copy(fuzzer_config, fuzzer_config_dir) for fuzzer in fuzzers: if fuzzers.count(fuzzer) > 1: raise Exception('Fuzzer "%s" provided more than once.' % fuzzer) validate_fuzzer(fuzzer) fuzzer_config_file_path = os.path.join(fuzzer_config_dir, fuzzer) # Create a simple yaml with just the fuzzer attribute. with open(fuzzer_config_file_path, 'w') as file_handle: file_handle.write('fuzzer: ' + fuzzer)
def test_recreate_directory_not_existing(tmp_path): """Tests that recreate_directory creates a directory that does not already exist.""" new_directory = os.path.join(tmp_path, 'new-directory') filesystem.recreate_directory(new_directory) assert os.path.exists(new_directory)
def test_recreate_directory_not_existing(fs): """Tests that recreate_directory creates a directory that does not already exist.""" new_directory = 'new-directory' filesystem.recreate_directory(new_directory) assert os.path.exists(new_directory)