class Benchmark: DATA_PATH = realpath("data") CHECKOUTS_PATH = realpath("checkouts") COMPILES_PATH = CHECKOUTS_PATH DETECTORS_PATH = realpath("detectors") FINDINGS_PATH = realpath("findings") DATASETS_FILE_PATH = join(DATA_PATH, 'datasets.yml') EX1_SUBFOLDER = "detect-only" EX2_SUBFOLDER = "mine-and-detect" EX3_SUBFOLDER = "mine-and-detect" def __init__(self, config): self.reviewed_eval_result_file = 'reviewed-result.csv' self.visualize_result_file = 'result.csv' self.config = config white_list = [] black_list = [] if 'white_list' in config: white_list.extend(config.white_list) if 'black_list' in config: black_list.extend(config.black_list) if 'dataset' in config: white_list.extend(get_white_list(join(self.DATASETS_FILE_PATH), config.dataset)) self.runner = TaskRunner(Benchmark.DATA_PATH, white_list, black_list) def _setup_stats(self) -> None: stats_calculator = stats.get_calculator(self.config.script) self.runner.add(stats_calculator) def _setup_info(self): self.runner.add(Info(Benchmark.CHECKOUTS_PATH, Benchmark.COMPILES_PATH)) def _setup_checkout(self): checkout_handler = Checkout(Benchmark.CHECKOUTS_PATH, self.config.force_checkout) self.runner.add(checkout_handler) def _setup_compile(self): compile_handler = Compile(Benchmark.CHECKOUTS_PATH, Benchmark.COMPILES_PATH, self.config.force_compile) self.runner.add(compile_handler) def _setup_detect(self): experiment = self.__get_experiment() self.runner.add(Detect(Benchmark.COMPILES_PATH, experiment, self.config.timeout, self.config.force_detect)) def _setup_publish_findings(self): experiment = self.__get_experiment() self.runner.add(PublishFindingsTask(experiment, self.config.dataset, self.config.review_site_url, self.config.review_site_user, self.config.review_site_password)) def _setup_publish_metadata(self): self.runner.add(PublishMetadataTask(self.config.review_site_url, self.config.review_site_user, self.config.review_site_password)) def __get_experiment(self): ex_ids = { 1: Experiment.PROVIDED_PATTERNS, 2: Experiment.TOP_FINDINGS, 3: Experiment.BENCHMARK } try: limit = self.config.limit except AttributeError: limit = 0 return Experiment(ex_ids.get(self.config.experiment), self.__get_detector(), Benchmark.FINDINGS_PATH, limit) def __get_detector(self): try: java_options = ['-' + option for option in self.config.java_options] return find_detector(self.DETECTORS_PATH, self.config.detector, java_options) except ValueError as e: logger.critical(e) exit() def run(self) -> None: if config.task == 'check': check_all_requirements() return elif config.task == 'info': self._setup_info() elif config.task == 'checkout': self._setup_checkout() elif config.task == 'compile': self._setup_checkout() self._setup_compile() elif config.task == 'detect': self._setup_checkout() self._setup_compile() self._setup_detect() elif config.task == 'publish': if config.publish_task == 'findings': self._setup_checkout() self._setup_compile() self._setup_detect() self._setup_publish_findings() elif config.publish_task == 'metadata': self._setup_checkout() self._setup_publish_metadata() elif config.task == 'stats': self._setup_stats() self.runner.run()
class TestTaskRunner: def setup(self): self.temp_dir = mkdtemp(prefix='mubench-datareader-test_') self.test_task = MagicMock() # type: ProjectTask self.uut = TaskRunner(self.temp_dir, white_list=[], black_list=[]) self.uut.add(self.test_task) def teardown(self): remove_tree(self.temp_dir) def test_processes_project(self): project = create_project("p1") self.uut._get_projects = MagicMock(return_value=[project]) self.uut.run() self.test_task.process_project.assert_called_with(project) def test_checks_requirements(self): requirement = Requirement("test requirement") requirement.check = MagicMock() self.test_task.get_requirements = MagicMock(return_value=[requirement]) self.uut.run() requirement.check.assert_called_with() def test_stops_on_unsatisfied_requirement(self): requirement = Requirement("test requirement") requirement.check = MagicMock(side_effect=ValueError("not satisfied")) self.test_task.get_requirements = MagicMock(return_value=[requirement]) with assert_raises(SystemExit): self.uut.run() # assert that exit comes before task is started self.test_task.start.assert_not_called() def test_starts_task(self): self.uut.run() self.test_task.start.assert_called_with() def test_ends_task(self): self.uut.run() self.test_task.end.assert_called_with() def test_finds_all_projects(self): p1 = create_project("p1", base_path=self.temp_dir) create_file(p1._project_file) p2 = create_project("p2", base_path=self.temp_dir) create_file(p2._project_file) self.uut.run() assert_equals([call(p1), call(p2)], self.test_task.process_project.call_args_list) def test_ignores_non_project_directories(self): create_file(join(self.temp_dir, "p1", "iamnotaproject.yml")) self.uut.run() self.test_task.process_project.assert_not_called() def test_skips_blacklisted_project(self): self.uut._get_projects = MagicMock(return_value=[create_project("p1")]) self.uut.black_list.append("p1") self.uut.run() self.test_task.process_project.assert_not_called() def test_runs_only_whitelisted_project(self): p2 = create_project("p2") self.uut._get_projects = MagicMock(return_value=[create_project("p1"), p2]) self.uut.white_list.append("p2") self.uut.run() assert_equals([call(p2)], self.test_task.process_project.call_args_list) def test_runs_whitelisted_project_if_version_only_whitelist(self): project = create_project("p") self.uut._get_projects = MagicMock(return_value=[project]) self.uut.white_list.append("p.42") self.uut.run() self.test_task.process_project.assert_called_with(project) def test_adds_project_to_backlist_when_task_answers_skip(self): self.uut._get_projects = MagicMock(return_value=[(create_project("p1"))]) self.test_task.process_project = MagicMock(return_value=["p1"]) self.uut.run() assert_equals(["p1"], self.uut.black_list)