def initialize_measurement_dirs(self): """Initialize directories that will be needed for measuring coverage.""" for directory in [ self.corpus_dir, self.coverage_dir, self.crashes_dir ]: filesystem.recreate_directory(directory) filesystem.create_directory(self.report_dir)
def generate_report(experiment_names, report_directory, report_name=None, label_by_experiment=False, benchmarks=None, fuzzers=None, report_type='default', quick=False, log_scale=False, from_cached_data=False, in_progress=False, end_time=None, merge_with_clobber=False): """Generate report helper.""" report_name = report_name or experiment_names[0] filesystem.create_directory(report_directory) data_path = os.path.join(report_directory, 'data.csv.gz') if from_cached_data and os.path.exists(data_path): experiment_df = pd.read_csv(data_path) else: experiment_df = queries.get_experiment_data(experiment_names) # Save the raw data along with the report. experiment_df.to_csv(data_path) data_utils.validate_data(experiment_df) if benchmarks is not None: experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks) if fuzzers is not None: experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers) if label_by_experiment: experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df) if end_time is not None: experiment_df = data_utils.filter_max_time(experiment_df, end_time) if merge_with_clobber: experiment_df = data_utils.clobber_experiments_data( experiment_df, experiment_names) fuzzer_names = experiment_df.fuzzer.unique() plotter = plotting.Plotter(fuzzer_names, quick, log_scale) experiment_ctx = experiment_results.ExperimentResults( experiment_df, report_directory, plotter, experiment_name=report_name) template = report_type + '.html' detailed_report = rendering.render_report(experiment_ctx, template, in_progress) filesystem.write(os.path.join(report_directory, 'index.html'), detailed_report)
def set_up_coverage_binaries(pool, experiment): """Set up coverage binaries for all benchmarks in |experiment|.""" # Use set comprehension to select distinct benchmarks. benchmarks = [ benchmark_tuple[0] for benchmark_tuple in db_utils.query(models.Trial.benchmark).distinct( ).filter(models.Trial.experiment == experiment) ] coverage_binaries_dir = build_utils.get_coverage_binaries_dir() filesystem.create_directory(coverage_binaries_dir) pool.map(set_up_coverage_binary, benchmarks)
def generate_coverage_regions_json(self): """Stores the coverage data in a json file.""" covered_regions = extract_covered_regions_from_summary_json( self.merged_summary_json_file) coverage_json_src = os.path.join(self.data_dir, 'covered_regions.json') coverage_json_dst = exp_path.filestore(coverage_json_src) filesystem.create_directory(self.data_dir) with open(coverage_json_src, 'w') as file_handle: json.dump(covered_regions, file_handle) filestore_utils.cp(coverage_json_src, coverage_json_dst, expect_zero=False)
def cp( # pylint: disable=invalid-name source, destination, recursive=False, parallel=False): # pylint: disable=unused-argument """Executes "cp" command from |source| to |destination|.""" # Create intermediate folders for `cp` command to behave like `gsutil.cp`. filesystem.create_directory(os.path.dirname(destination)) command = ['cp'] if recursive: command.append('-r') command.extend([source, destination]) return new_process.execute(command, expect_zero=True)
def set_up_coverage_binary(benchmark): """Set up coverage binaries for |benchmark|.""" initialize_logs() coverage_binaries_dir = build_utils.get_coverage_binaries_dir() benchmark_coverage_binary_dir = coverage_binaries_dir / benchmark filesystem.create_directory(benchmark_coverage_binary_dir) archive_name = 'coverage-build-%s.tar.gz' % benchmark archive_filestore_path = exp_path.filestore(coverage_binaries_dir / archive_name) filestore_utils.cp(archive_filestore_path, str(benchmark_coverage_binary_dir)) archive_path = benchmark_coverage_binary_dir / archive_name tar = tarfile.open(archive_path, 'r:gz') tar.extractall(benchmark_coverage_binary_dir) os.remove(archive_path)
def _add_build_arguments_to_config(base: str, fuzzer: str) -> str: """If there are fuzzer-specific arguments, make a config file with them.""" fuzzer_config = fuzzer_config_utils.get_by_variant_name(fuzzer) if 'build_arguments' not in fuzzer_config: return base # TODO(mbarbella): Rather than rewrite yaml files, use the GCB API. args = fuzzer_config['build_arguments'] config = yaml_utils.read(base) for step in config['steps']: if 'id' in step and step['id'] in BUILDER_STEP_IDS: # Append additional flags before the final argument. step['args'] = step['args'][:-1] + args + [step['args'][-1]] new_config_path = os.path.join(CONFIG_DIR, 'builds', fuzzer + '.yaml') filesystem.create_directory(os.path.dirname(new_config_path)) yaml_utils.write(new_config_path, config) return new_config_path
def generate_report(experiment_names, report_directory, report_name=None, label_by_experiment=False, fuzzers=None, report_type='default', quick=False, from_cached_data=False): """Generate report helper.""" report_name = report_name or experiment_names[0] filesystem.create_directory(report_directory) data_path = os.path.join(report_directory, 'data.csv.gz') if from_cached_data and os.path.exists(data_path): experiment_df = pd.read_csv(data_path) else: experiment_df = queries.get_experiment_data(experiment_names) # Save the raw data along with the report. experiment_df.to_csv(data_path) if fuzzers is not None: experiment_df = filter_fuzzers(experiment_df, fuzzers) if label_by_experiment: experiment_df = label_fuzzers_by_experiment(experiment_df) fuzzer_names = experiment_df.fuzzer.unique() plotter = plotting.Plotter(fuzzer_names, quick) experiment_ctx = experiment_results.ExperimentResults( experiment_df, report_directory, plotter, experiment_name=report_name) template = report_type + '.html' detailed_report = rendering.render_report(experiment_ctx, template) filesystem.write(os.path.join(report_directory, 'index.html'), detailed_report)
def start(self): """Start the experiment on the dispatcher.""" container_name = 'dispatcher-container' logs.info('Started dispatcher with container name: %s', container_name) experiment_filestore_path = os.path.abspath( self.config['experiment_filestore']) filesystem.create_directory(experiment_filestore_path) sql_database_arg = ( 'SQL_DATABASE_URL=sqlite:///{}?check_same_thread=False'.format( os.path.join(experiment_filestore_path, 'local.db'))) docker_registry = self.config['docker_registry'] set_instance_name_arg = 'INSTANCE_NAME={instance_name}'.format( instance_name=self.instance_name) set_experiment_arg = 'EXPERIMENT={experiment}'.format( experiment=self.config['experiment']) shared_experiment_filestore_arg = '{0}:{0}'.format( self.config['experiment_filestore']) # TODO: (#484) Use config in function args or set as environment # variables. set_docker_registry_arg = 'DOCKER_REGISTRY={}'.format(docker_registry) set_experiment_filestore_arg = ( 'EXPERIMENT_FILESTORE={experiment_filestore}'.format( experiment_filestore=self.config['experiment_filestore'])) shared_report_filestore_arg = '{0}:{0}'.format( self.config['report_filestore']) set_report_filestore_arg = ( 'REPORT_FILESTORE={report_filestore}'.format( report_filestore=self.config['report_filestore'])) docker_image_url = '{docker_registry}/dispatcher-image'.format( docker_registry=docker_registry) command = [ 'docker', 'run', '-ti', '--rm', '-v', '/var/run/docker.sock:/var/run/docker.sock', '-v', shared_experiment_filestore_arg, '-v', shared_report_filestore_arg, '-e', set_instance_name_arg, '-e', set_experiment_arg, '-e', sql_database_arg, '-e', set_experiment_filestore_arg, '-e', set_report_filestore_arg, '-e', set_docker_registry_arg, '-e', 'LOCAL_EXPERIMENT=True', '--cap-add=SYS_PTRACE', '--cap-add=SYS_NICE', '--name=%s' % container_name, docker_image_url, '/bin/bash', '-c', 'rsync -r ' '"${EXPERIMENT_FILESTORE}/${EXPERIMENT}/input/" ${WORK} && ' 'mkdir ${WORK}/src && ' 'tar -xvzf ${WORK}/src.tar.gz -C ${WORK}/src && ' 'source "${WORK}/.venv/bin/activate" && ' 'pip3 install -r "${WORK}/src/requirements.txt" && ' 'PYTHONPATH=${WORK}/src python3 ' '${WORK}/src/experiment/dispatcher.py || ' '/bin/bash' # Open shell if experiment fails. ] return new_process.execute(command, write_to_stdout=True)
def generate_report(experiment_names, report_directory, report_name=None, label_by_experiment=False, benchmarks=None, fuzzers=None, report_type='default', quick=False, log_scale=False, from_cached_data=False, in_progress=False, end_time=None, merge_with_clobber=False, merge_with_clobber_nonprivate=False, coverage_report=False): """Generate report helper.""" if merge_with_clobber_nonprivate: experiment_names = ( queries.add_nonprivate_experiments_for_merge_with_clobber( experiment_names)) main_experiment_name = experiment_names[0] report_name = report_name or main_experiment_name filesystem.create_directory(report_directory) data_path = os.path.join(report_directory, 'data.csv.gz') if from_cached_data and os.path.exists(data_path): experiment_df = pd.read_csv(data_path) description = "from cached data" else: experiment_df = queries.get_experiment_data(experiment_names) description = queries.get_experiment_description(main_experiment_name) data_utils.validate_data(experiment_df) if benchmarks is not None: experiment_df = data_utils.filter_benchmarks(experiment_df, benchmarks) if fuzzers is not None: experiment_df = data_utils.filter_fuzzers(experiment_df, fuzzers) if label_by_experiment: experiment_df = data_utils.label_fuzzers_by_experiment(experiment_df) if end_time is not None: experiment_df = data_utils.filter_max_time(experiment_df, end_time) if merge_with_clobber or merge_with_clobber_nonprivate: experiment_df = data_utils.clobber_experiments_data( experiment_df, experiment_names) # Save the filtered raw data along with the report if not using cached data # or if the data does not exist. if not from_cached_data or not os.path.exists(data_path): experiment_df.to_csv(data_path) # Load the coverage json summary file. coverage_dict = {} if coverage_report: coverage_dict = coverage_data_utils.get_covered_regions_dict( experiment_df) fuzzer_names = experiment_df.fuzzer.unique() plotter = plotting.Plotter(fuzzer_names, quick, log_scale) experiment_ctx = experiment_results.ExperimentResults( experiment_df, coverage_dict, report_directory, plotter, experiment_name=report_name) template = report_type + '.html' detailed_report = rendering.render_report(experiment_ctx, template, in_progress, coverage_report, description) filesystem.write(os.path.join(report_directory, 'index.html'), detailed_report)
def start(self): """Start the experiment on the dispatcher.""" experiment_filestore_path = os.path.abspath( self.config['experiment_filestore']) filesystem.create_directory(experiment_filestore_path) sql_database_arg = 'SQL_DATABASE_URL=sqlite:///{}'.format( os.path.join(experiment_filestore_path, 'local.db')) base_docker_tag = experiment_utils.get_base_docker_tag( self.config['cloud_project']) set_instance_name_arg = 'INSTANCE_NAME={instance_name}'.format( instance_name=self.instance_name) set_experiment_arg = 'EXPERIMENT={experiment}'.format( experiment=self.config['experiment']) set_cloud_project_arg = 'CLOUD_PROJECT={cloud_project}'.format( cloud_project=self.config['cloud_project']) shared_experiment_filestore_arg = '{0}:{0}'.format( self.config['experiment_filestore']) set_experiment_filestore_arg = ( 'EXPERIMENT_FILESTORE={experiment_filestore}'.format( experiment_filestore=self.config['experiment_filestore'])) shared_report_filestore_arg = '{0}:{0}'.format( self.config['report_filestore']) set_report_filestore_arg = ( 'REPORT_FILESTORE={report_filestore}'.format( report_filestore=self.config['report_filestore'])) docker_image_url = '{base_docker_tag}/dispatcher-image'.format( base_docker_tag=base_docker_tag) command = [ 'docker', 'run', '-ti', '--rm', '-v', '/var/run/docker.sock:/var/run/docker.sock', '-v', shared_experiment_filestore_arg, '-v', shared_report_filestore_arg, '-e', set_instance_name_arg, '-e', set_experiment_arg, '-e', set_cloud_project_arg, '-e', sql_database_arg, '-e', set_experiment_filestore_arg, '-e', set_report_filestore_arg, '-e', 'LOCAL_EXPERIMENT=True', '--cap-add=SYS_PTRACE', '--cap-add=SYS_NICE', '--name=dispatcher-container', docker_image_url, '/bin/bash', '-c', 'rsync -r ' '"${EXPERIMENT_FILESTORE}/${EXPERIMENT}/input/" ${WORK} && ' 'mkdir ${WORK}/src && ' 'tar -xvzf ${WORK}/src.tar.gz -C ${WORK}/src && ' 'source "${WORK}/.venv/bin/activate" && ' 'pip3 install -r "${WORK}/src/requirements.txt" && ' 'PYTHONPATH=${WORK}/src python3 ' '${WORK}/src/experiment/dispatcher.py || ' '/bin/bash' # Open shell if experiment fails. ] return new_process.execute(command, write_to_stdout=True)
def generate_report(experiment_names, report_directory, report_name=None, label_by_experiment=False, benchmarks=None, fuzzers=None, report_type='default', quick=False, log_scale=False, from_cached_data=False, in_progress=False, end_time=None, merge_with_clobber=False, merge_with_clobber_nonprivate=False, coverage_report=False): """Generate report helper.""" if merge_with_clobber_nonprivate: experiment_names = ( queries.add_nonprivate_experiments_for_merge_with_clobber( experiment_names)) merge_with_clobber = True main_experiment_name = experiment_names[0] report_name = report_name or main_experiment_name filesystem.create_directory(report_directory) data_path = os.path.join(report_directory, DATA_FILENAME) experiment_df, experiment_description = get_experiment_data( experiment_names, main_experiment_name, from_cached_data, data_path) # TODO(metzman): Ensure that each experiment is in the df. Otherwise there # is a good chance user misspelled something. data_utils.validate_data(experiment_df) experiment_df = modify_experiment_data_if_requested( experiment_df, experiment_names, benchmarks, fuzzers, label_by_experiment, end_time, merge_with_clobber) # Add |bugs_covered| column prior to export. experiment_df = data_utils.add_bugs_covered_column(experiment_df) # Save the filtered raw data along with the report if not using cached data # or if the data does not exist. if not from_cached_data or not os.path.exists(data_path): experiment_df.to_csv(data_path) # Load the coverage json summary file. coverage_dict = {} if coverage_report: logger.info('Generating coverage report info.') coverage_dict = coverage_data_utils.get_covered_regions_dict( experiment_df) logger.info('Finished generating coverage report info.') fuzzer_names = experiment_df.fuzzer.unique() plotter = plotting.Plotter(fuzzer_names, quick, log_scale) experiment_ctx = experiment_results.ExperimentResults( experiment_df, coverage_dict, report_directory, plotter, experiment_name=report_name) template = report_type + '.html' logger.info('Rendering HTML report.') detailed_report = rendering.render_report(experiment_ctx, template, in_progress, coverage_report, experiment_description) logger.info('Done rendering HTML report.') filesystem.write(os.path.join(report_directory, 'index.html'), detailed_report)