class Benchmark:
DATA_PATH = realpath("data")
CHECKOUTS_PATH = realpath("checkouts")
COMPILES_PATH = CHECKOUTS_PATH
DETECTORS_PATH = realpath("detectors")
FINDINGS_PATH = realpath("findings")
DATASETS_FILE_PATH = join(DATA_PATH, 'datasets.yml')
EX1_SUBFOLDER = "detect-only"
EX2_SUBFOLDER = "mine-and-detect"
EX3_SUBFOLDER = "mine-and-detect"
def __init__(self, config):
self.reviewed_eval_result_file = 'reviewed-result.csv'
self.visualize_result_file = 'result.csv'
self.config = config
white_list = []
black_list = []
if 'white_list' in config:
white_list.extend(config.white_list)
if 'black_list' in config:
black_list.extend(config.black_list)
if 'dataset' in config:
white_list.extend(get_white_list(join(self.DATASETS_FILE_PATH), config.dataset))
self.runner = TaskRunner(Benchmark.DATA_PATH, white_list, black_list)
def _setup_stats(self) -> None:
stats_calculator = stats.get_calculator(self.config.script)
self.runner.add(stats_calculator)
def _setup_info(self):
self.runner.add(Info(Benchmark.CHECKOUTS_PATH, Benchmark.COMPILES_PATH))
def _setup_checkout(self):
checkout_handler = Checkout(Benchmark.CHECKOUTS_PATH, self.config.force_checkout)
self.runner.add(checkout_handler)
def _setup_compile(self):
compile_handler = Compile(Benchmark.CHECKOUTS_PATH, Benchmark.COMPILES_PATH, self.config.force_compile)
self.runner.add(compile_handler)
def _setup_detect(self):
experiment = self.__get_experiment()
self.runner.add(Detect(Benchmark.COMPILES_PATH, experiment, self.config.timeout, self.config.force_detect))
def _setup_publish_findings(self):
experiment = self.__get_experiment()
self.runner.add(PublishFindingsTask(experiment, self.config.dataset, self.config.review_site_url,
self.config.review_site_user, self.config.review_site_password))
def _setup_publish_metadata(self):
self.runner.add(PublishMetadataTask(self.config.review_site_url,
self.config.review_site_user, self.config.review_site_password))
def __get_experiment(self):
ex_ids = {
1: Experiment.PROVIDED_PATTERNS,
2: Experiment.TOP_FINDINGS,
3: Experiment.BENCHMARK
}
try:
limit = self.config.limit
except AttributeError:
limit = 0
return Experiment(ex_ids.get(self.config.experiment), self.__get_detector(), Benchmark.FINDINGS_PATH, limit)
def __get_detector(self):
try:
java_options = ['-' + option for option in self.config.java_options]
return find_detector(self.DETECTORS_PATH, self.config.detector, java_options)
except ValueError as e:
logger.critical(e)
exit()
def run(self) -> None:
if config.task == 'check':
check_all_requirements()
return
elif config.task == 'info':
self._setup_info()
elif config.task == 'checkout':
self._setup_checkout()
elif config.task == 'compile':
self._setup_checkout()
self._setup_compile()
elif config.task == 'detect':
self._setup_checkout()
self._setup_compile()
self._setup_detect()
elif config.task == 'publish':
if config.publish_task == 'findings':
self._setup_checkout()
self._setup_compile()
self._setup_detect()
self._setup_publish_findings()
elif config.publish_task == 'metadata':
self._setup_checkout()
self._setup_publish_metadata()
elif config.task == 'stats':
self._setup_stats()
self.runner.run()
class TestTaskRunner:
def setup(self):
self.temp_dir = mkdtemp(prefix='mubench-datareader-test_')
self.test_task = MagicMock() # type: ProjectTask
self.uut = TaskRunner(self.temp_dir, white_list=[], black_list=[])
self.uut.add(self.test_task)
def teardown(self):
remove_tree(self.temp_dir)
def test_processes_project(self):
project = create_project("p1")
self.uut._get_projects = MagicMock(return_value=[project])
self.uut.run()
self.test_task.process_project.assert_called_with(project)
def test_checks_requirements(self):
requirement = Requirement("test requirement")
requirement.check = MagicMock()
self.test_task.get_requirements = MagicMock(return_value=[requirement])
self.uut.run()
requirement.check.assert_called_with()
def test_stops_on_unsatisfied_requirement(self):
requirement = Requirement("test requirement")
requirement.check = MagicMock(side_effect=ValueError("not satisfied"))
self.test_task.get_requirements = MagicMock(return_value=[requirement])
with assert_raises(SystemExit):
self.uut.run()
# assert that exit comes before task is started
self.test_task.start.assert_not_called()
def test_starts_task(self):
self.uut.run()
self.test_task.start.assert_called_with()
def test_ends_task(self):
self.uut.run()
self.test_task.end.assert_called_with()
def test_finds_all_projects(self):
p1 = create_project("p1", base_path=self.temp_dir)
create_file(p1._project_file)
p2 = create_project("p2", base_path=self.temp_dir)
create_file(p2._project_file)
self.uut.run()
assert_equals([call(p1), call(p2)], self.test_task.process_project.call_args_list)
def test_ignores_non_project_directories(self):
create_file(join(self.temp_dir, "p1", "iamnotaproject.yml"))
self.uut.run()
self.test_task.process_project.assert_not_called()
def test_skips_blacklisted_project(self):
self.uut._get_projects = MagicMock(return_value=[create_project("p1")])
self.uut.black_list.append("p1")
self.uut.run()
self.test_task.process_project.assert_not_called()
def test_runs_only_whitelisted_project(self):
p2 = create_project("p2")
self.uut._get_projects = MagicMock(return_value=[create_project("p1"), p2])
self.uut.white_list.append("p2")
self.uut.run()
assert_equals([call(p2)], self.test_task.process_project.call_args_list)
def test_runs_whitelisted_project_if_version_only_whitelist(self):
project = create_project("p")
self.uut._get_projects = MagicMock(return_value=[project])
self.uut.white_list.append("p.42")
self.uut.run()
self.test_task.process_project.assert_called_with(project)
def test_adds_project_to_backlist_when_task_answers_skip(self):
self.uut._get_projects = MagicMock(return_value=[(create_project("p1"))])
self.test_task.process_project = MagicMock(return_value=["p1"])
self.uut.run()
assert_equals(["p1"], self.uut.black_list)