def setup(self):
self.dataset = "-d-"
self.project = create_project("-p-")
self.misuse = create_misuse("-m-", project=self.project)
self.version = create_version("-v-", project=self.project, misuses=[self.misuse])
self.test_run = Run([MagicMock()])
self.test_run.is_success = lambda: False
self.test_run.is_error = lambda: False
self.test_run.is_timeout = lambda: False
self.detector = MagicMock() # type: Detector
self.detector.id = "test_detector"
self.experiment = MagicMock() # type: Experiment
self.experiment.id = "test_experiment"
self.experiment.get_run = lambda v: self.test_run
self.experiment.detector = self.detector
self.uut = PublishFindingsTask(self.experiment, self.dataset, "http://dummy.url", "-username-")
class TestPublishFindingsTask:
# noinspection PyAttributeOutsideInit
def setup(self):
self.dataset = "-d-"
self.project = create_project("-p-")
self.misuse = create_misuse("-m-", project=self.project)
self.version = create_version("-v-", project=self.project, misuses=[self.misuse])
self.test_run = Run([MagicMock()])
self.test_run.is_success = lambda: False
self.test_run.is_error = lambda: False
self.test_run.is_timeout = lambda: False
self.detector = MagicMock() # type: Detector
self.detector.id = "test_detector"
self.experiment = MagicMock() # type: Experiment
self.experiment.id = "test_experiment"
self.experiment.get_run = lambda v: self.test_run
self.experiment.detector = self.detector
self.uut = PublishFindingsTask(self.experiment, self.dataset, "http://dummy.url", "-username-")
def test_post_url(self, post_mock):
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[0][0], "http://dummy.url/api/upload/" + self.experiment.id)
@patch("benchmark.tasks.implementations.publish_findings_task.getpass.getpass")
def test_post_auth_prompt(self, pass_mock, post_mock):
pass_mock.return_value = "-password-"
self.uut.start() # asks for password once on start
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[1]["username"], "-username-")
assert_equals(post_mock.call_args[1]["password"], "-password-")
@patch("benchmark.tasks.implementations.publish_findings_task.getpass.getpass")
def test_post_auth_provided(self, pass_mock, post_mock):
pass_mock.side_effect = UserWarning("should skip prompt")
self.uut.review_site_password = "******"
self.uut.start() # should not ask for a password, since already set
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[1]["username"], "-username-")
assert_equals(post_mock.call_args[1]["password"], "-password-")
def test_publish_successful_run(self, post_mock):
self.test_run.is_success = lambda: True
self.test_run.get_runtime = lambda: 42
findings = [
_create_finding({"id": "-1-", "misuse": "-p-.-m1-", "detector_specific": "-specific1-"}),
_create_finding({"id": "-2-", "misuse": "-p-.-m2-", "detector_specific": "-specific2-"})
]
self.test_run.get_findings = lambda: findings
potential_hits = findings[:1]
self.test_run.get_potential_hits = lambda: potential_hits
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[0][1], {
"dataset": self.dataset,
"detector": self.detector.id,
"project": self.project.id,
"version": self.version.version_id,
"result": "success",
"runtime": 42.0,
"number_of_findings": 2,
"potential_hits": potential_hits
})
def test_publish_successful_run_files(self, post_mock):
self.test_run.is_success = lambda: True
self.test_run.get_potential_hits = lambda: [
_create_finding({"id": "-1-"}, file_paths=["-file1-"]),
_create_finding({"id": "-2-"}, file_paths=["-file2-"])
]
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[1]["file_paths"], ["-file1-", "-file2-"])
def test_publish_successful_run_in_chunks(self, post_mock):
self.uut.max_files_per_post = 1
self.test_run.is_success = lambda: True
self.test_run.get_potential_hits = lambda: [
_create_finding({"id": "-1-"}, file_paths=["-file1-"]),
_create_finding({"id": "-2-"}, file_paths=["-file2-"])
]
self.uut.process_project_version(self.project, self.version)
assert_equals(len(post_mock.call_args_list), 2)
assert_equals(post_mock.call_args_list[0][1]["file_paths"], ["-file1-"])
assert_equals(post_mock.call_args_list[1][1]["file_paths"], ["-file2-"])
def test_publish_successful_run_in_partial_chunks(self, post_mock):
self.uut.max_files_per_post = 3
self.test_run.is_success = lambda: True
self.test_run.get_potential_hits = lambda: [
_create_finding({"id": "-1-"}, file_paths=["-file1-", "-file2-"]),
_create_finding({"id": "-2-"}, file_paths=["-file3-", "-file4-"])
]
self.uut.process_project_version(self.project, self.version)
assert_equals(len(post_mock.call_args_list), 2)
def test_publish_successful_run_code_snippets(self, post_mock):
self.test_run.is_success = lambda: True
self.test_run.get_potential_hits = lambda: [_create_finding({"id": "42"}, snippets=[Snippet("-code-", 23)])]
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[0][1]["potential_hits"][0]["target_snippets"],
[{"code": "-code-", "first_line_number": 23}])
def test_publish_erroneous_run(self, post_mock):
self.test_run.is_error = lambda: True
self.test_run.get_runtime = lambda: 1337
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[0][1], {
"dataset": self.dataset,
"detector": self.detector.id,
"project": self.project.id,
"version": self.version.version_id,
"result": "error",
"runtime": 1337,
"number_of_findings": 0,
"potential_hits": []
})
def test_publish_timeout_run(self, post_mock):
self.test_run.is_timeout = lambda: True
self.test_run.get_runtime = lambda: 1000000
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[0][1], {
"dataset": self.dataset,
"detector": self.detector.id,
"project": self.project.id,
"version": self.version.version_id,
"result": "timeout",
"runtime": 1000000,
"number_of_findings": 0,
"potential_hits": []
})
def test_publish_not_run(self, post_mock):
self.test_run.get_runtime = lambda: 0
self.uut.process_project_version(self.project, self.version)
assert_equals(post_mock.call_args[0][1], {
"dataset": self.dataset,
"detector": self.detector.id,
"project": self.project.id,
"version": self.version.version_id,
"result": "not run",
"runtime": 0,
"number_of_findings": 0,
"potential_hits": []
})
def test_nothing_to_upload(self, post_mock):
self.uut.end()
post_mock.assert_not_called()
