class CheckmarxTest(unittest.TestCase):
""" Unit tests for the Checkmarx class. """
def setUp(self):
self.__opener = FakeUrlOpener()
self.__report = Checkmarx('http://url',
'username',
'password',
url_open=self.__opener)
def test_high_risk_warnings(self):
""" Test the number of high risk warnings. """
self.assertEqual(0, self.__report.nr_warnings(['id'], 'high'))
def test_medium_risk_warnins(self):
""" Test the number of medium risk warnings. """
self.assertEqual(1, self.__report.nr_warnings(['id'], 'medium'))
def test_passed_raise(self):
""" Test that the value is -1 when the report can't be opened. """
self.assertEqual(-1, self.__report.nr_warnings(['raise'], 'high'))
def test_multiple_urls(self):
""" Test the number of alerts for multiple urls. """
self.assertEqual(2, self.__report.nr_warnings(['id1', 'id2'],
'medium'))
def test_metric_source_urls_without_report(self):
""" Test the metric source urls without metric ids. """
self.assertEqual([], self.__report.metric_source_urls())
def test_metric_source_urls(self):
""" Test the metric source urls with one metric id. """
self.assertEqual([], self.__report.metric_source_urls('id'))
def test_metric_source_urls_on_error(self):
""" Test the metric source urls when an error occurs. """
self.assertEqual(["http://url/"],
self.__report.metric_source_urls('raise'))
def test_url(self):
""" Test the metric source base url. """
self.assertEqual("http://url/", self.__report.url())
def test_datetime(self):
""" Test the date and time of the report. """
self.assertEqual(datetime.datetime(2017, 9, 20, 0, 43, 35),
self.__report.datetime('id'))
def test_datetime_missing(self):
""" Test a missing date and time of the report. """
self.__opener.json = '{}'
self.assertEqual(datetime.datetime.min, self.__report.datetime('id'))
def test_datetime_when_code_unchanged(self):
""" Test that the date and time of the report is the date and time of the last check when code is unchanged. """
self.__opener.json = '{"value": [{"LastScan": {"ScanCompletedOn": "2017-09-20T00:43:35.73+01:00", ' \
'"Comment": "Attempt to perform scan on 9/26/2017 12:30:24 PM - No code changes ' \
'were detected; "}}]}'
self.assertEqual(datetime.datetime(2017, 9, 26, 12, 30, 24),
self.__report.datetime('id'))
def test_datetime_when_code_unchanged_multiple_times(self):
""" Test that the date and time of the report is the date and time of the last check when code is unchanged. """
self.__opener.json = '{"value": [{"LastScan": {"ScanCompletedOn": "2017-09-20T00:43:35.73+01:00", ' \
'"Comment": "Attempt to perform scan on 9/26/2017 12:30:24 PM - No code changes ' \
'were detected; Attempt to perform scan on 9/27/2017 12:30:24 PM - No code changes ' \
'were detected; "}}]}'
self.assertEqual(datetime.datetime(2017, 9, 27, 12, 30, 24),
self.__report.datetime('id'))
def test_datetime_when_some_checks_have_no_date(self):
""" Test that the date and time of the report is the date and time of the last check when code is unchanged. """
self.__opener.json = '{"value": [{"LastScan": {"ScanCompletedOn": "2016-12-14T00:01:30.737+01:00", ' \
'"Comment": "Attempt to perform scan on 2/13/2017 8:00:06 PM - No code changes were ' \
'detected; No code changes were detected No code changes were detected"}}]}'
self.assertEqual(datetime.datetime(2017, 2, 13, 20, 0, 6),
self.__report.datetime('id'))
def test_nr_warnings_on_missing_values(self):
""" Test dealing with empty list of values. """
self.__opener.json = '{"value": []}'
self.assertEqual(-1, self.__report.nr_warnings(['id'], 'medium'))
def test_datetime_on_missing_values(self):
""" Test dealing with empty list of values. """
self.__opener.json = '{"value": []}'
self.assertEqual(datetime.datetime.min, self.__report.datetime('id'))
def test_datetime_on_url_exception(self):
""" Test dealing with empty list of values. """
self.__opener.json = '{"value": []}'
self.assertEqual(datetime.datetime.min,
self.__report.datetime('raise'))
class CheckmarxTest(unittest.TestCase):
""" Unit tests for the Checkmarx class. """
# pylint: disable=too-many-public-methods
def setUp(self):
time.sleep = MagicMock()
# pylint: disable=protected-access
Checkmarx._fetch_project_id.cache_clear()
with patch.object(url_opener.UrlOpener,
'url_read',
return_value='{"access_token": "abc123"}'):
self.__report = Checkmarx('http://url', 'username', 'password')
def test_high_risk_warnings(self, mock_url_read):
""" Test the number of high risk warnings. """
mock_url_read.side_effect = [PROJECTS, LAST_SCAN, STATISTICS]
self.assertEqual(
4, self.__report.nr_warnings(['metric_source_id'], 'high'))
self.assertEqual(mock_url_read.call_args_list[0][0][0],
'http://url/CxRestAPI/projects')
self.assertEqual(
mock_url_read.call_args_list[1][0][0],
'http://url/CxRestAPI/sast/scans?projectId=11&last=1')
self.assertEqual(
mock_url_read.call_args_list[2][0][0],
'http://url/CxRestAPI/sast/scans/10111/resultsStatistics')
@patch.object(logging, 'error')
def test_nr_warnings_no_project(self, mock_error, mock_url_read):
""" Test the number of high risk warnings. """
mock_url_read.return_value = PROJECTS
self.assertEqual(
-1, self.__report.nr_warnings(['unknown_proj_id'], 'high'))
mock_error.assert_called_once_with(
"Error: no project id found for project with name '%s'.",
'unknown_proj_id')
def test_obtain_issues(self, mock_url_read):
""" Test that issues are correctly obtained. """
mock_url_read.side_effect = [
PROJECTS, LAST_SCAN, '{"reportId": 22}',
'{"status": {"value": "Created"}}',
SAST_REPORT.format(false_positive=False, severity='High')
]
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertIsInstance(issues[0], Checkmarx.Issue)
self.assertEqual('JScript Vulnerabilities', issues[0].group)
self.assertEqual('Reflected XSS', issues[0].title)
self.assertEqual(
'http://url/CxWebClient/ScanQueryDescription.aspx?queryID=789&'
'queryVersionCode=842956&queryTitle=Reflected_XSS',
issues[0].display_url)
self.assertEqual(1, issues[0].count)
self.assertEqual("Recurrent", issues[0].status)
self.assertEqual(mock_url_read.call_args_list[0][0][0],
'http://url/CxRestAPI/projects')
self.assertEqual(
mock_url_read.call_args_list[1][0][0],
'http://url/CxRestAPI/sast/scans?projectId=11&last=1')
self.assertEqual(mock_url_read.call_args_list[2][0][0],
'http://url/CxRestAPI/reports/sastScan')
self.assertEqual(mock_url_read.call_args_list[3][0][0],
'http://url/CxRestAPI/reports/sastScan/22/status')
self.assertEqual(mock_url_read.call_args_list[4][0][0],
'http://url/CxRestAPI/reports/sastScan/22')
@patch.object(logging, 'error')
def test_obtain_issues_xml_error(self, mock_error, mock_url_read):
""" Test that issues are correctly obtained. """
mock_url_read.side_effect = \
[PROJECTS, LAST_SCAN, '{"reportId": 22}', '{"status": {"value": "Created"}}', 'not-an-xml']
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
self.assertEqual(mock_error.call_args[0][0],
"Error in checkmarx report xml: %s.")
self.assertIsInstance(mock_error.call_args[0][1],
xml.etree.ElementTree.ParseError)
@patch.object(logging, 'error')
def test_obtain_issue_ssast_report_not_created(self, mock_error,
mock_url_read):
""" Test that issues are correctly obtained. """
mock_url_read.side_effect = [PROJECTS, LAST_SCAN, '{"reportId": 22}'] + \
['{"status": {"value": "InProgress"}}'] * 10
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
mock_error.assert_called_once_with(
"SAST report is not created on the Checkmarx server!")
@patch.object(logging, 'error')
def test_obtain_issues_xml_tag_error(self, mock_error, mock_url_read):
""" Test that issues are correctly obtained. """
mock_url_read.side_effect = [
PROJECTS, LAST_SCAN, '{"reportId": 22}',
'{"status": {"value": "Created"}}',
'<CxXMLResults><Query /></CxXMLResults>'
]
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
self.assertEqual(mock_error.call_args[0][0],
"Tag %s could not be found.")
self.assertIsInstance(mock_error.call_args[0][1], KeyError)
def test_obtain_issues_exclude_false_positives(self, mock_url_read):
""" Test that issues are omitted when false positive. """
mock_url_read.side_effect = [
PROJECTS, LAST_SCAN, '{"reportId": 22}',
'{"status": {"value": "Created"}}',
SAST_REPORT.format(false_positive=True, severity='High')
]
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
def test_obtain_issues_exclude_wrong_severity(self, mock_url_read):
""" Test that issues are omitted when severity does not match. """
mock_url_read.side_effect = [
PROJECTS, LAST_SCAN, '{"reportId": 22}',
'{"status": {"value": "Created"}}',
SAST_REPORT.format(false_positive=False, severity='Low')
]
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
def test_obtain_issues_no_query(self, mock_url_read):
""" Test that issues are omitted when there is no query. """
mock_url_read.side_effect = \
[PROJECTS, LAST_SCAN, '{"reportId": 22}', '{"status": {"value": "Created"}}', '<CxXMLResults />']
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
def test_obtain_issues_http_error(self, mock_url_read):
""" Test that issues are omitted when http error occurs. """
mock_url_read.side_effect = urllib.error.HTTPError(
'raise', None, None, None, None)
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
@patch.object(logging, 'error')
def test_obtain_issues_response_error(self, mock_error, mock_url_read):
""" Test that issues are omitted when json error occurs. """
mock_url_read.return_value = 'non-json'
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
self.assertEqual(mock_error.call_args[0][0], "Error loading json: %s.")
self.assertIsInstance(mock_error.call_args[0][1], ValueError)
@patch.object(logging, 'error')
def test_obtain_issues_json_error(self, mock_error, mock_url_read):
""" Test that issues are omitted when json error occurs. """
mock_url_read.side_effect = [PROJECTS, '{}']
self.__report.obtain_issues(['metric_source_id'], 'high')
issues = self.__report.issues()
self.assertIsInstance(issues, List)
self.assertEqual(len(issues), 0)
self.assertEqual(mock_error.call_args[0][0],
"Tag %s could not be found.")
self.assertIsInstance(mock_error.call_args[0][1], KeyError)
def test_medium_risk_warnings(self, mock_url_read):
""" Test the number of medium risk warnings. """
mock_url_read.side_effect = [PROJECTS, LAST_SCAN, STATISTICS]
self.assertEqual(
7, self.__report.nr_warnings(['metric_source_id'], 'medium'))
def test_passed_raise(self, mock_url_read):
""" Test that the value is -1 when the report can't be opened. """
mock_url_read.side_effect = urllib.error.HTTPError(
'raise', None, None, None, None)
self.assertEqual(-1, self.__report.nr_warnings(['raise'], 'high'))
mock_url_read.assert_called_once_with('http://url/CxRestAPI/projects')
def test_multiple_urls(self, mock_url_read):
""" Test the number of alerts for multiple urls. """
mock_url_read.side_effect = [
PROJECTS, LAST_SCAN, STATISTICS, PROJECTS, '[{"id": 202222}]',
STATISTICS
]
self.assertEqual(
14, self.__report.nr_warnings(['metric_source_id', 'id2'],
'medium'))
self.assertEqual([
call('http://url/CxRestAPI/projects'),
call('http://url/CxRestAPI/sast/scans?projectId=11&last=1'),
call('http://url/CxRestAPI/sast/scans/10111/resultsStatistics'),
call('http://url/CxRestAPI/projects'),
call('http://url/CxRestAPI/sast/scans?projectId=22&last=1'),
call('http://url/CxRestAPI/sast/scans/202222/resultsStatistics')
], mock_url_read.call_args_list)
def test_metric_source_urls_without_report(self, mock_url_read):
""" Test the metric source urls without metric ids. """
mock_url_read.return_value = None
self.assertEqual([], self.__report.metric_source_urls())
def test_metric_source_urls(self, mock_url_read):
""" Test the metric source urls with one metric id. """
mock_url_read.side_effect = [PROJECTS, LAST_SCAN]
self.assertEqual([
'http://url/CxWebClient/ViewerMain.aspx?scanId=10111&ProjectID=22'
], self.__report.metric_source_urls('id2'))
@patch.object(logging, 'error')
def test_metric_source_urls_key_error(self, mock_error, mock_url_read):
""" Test the metric source urls with empty scan response.. """
mock_url_read.side_effect = [PROJECTS, '{}']
self.assertEqual(["http://url/"],
self.__report.metric_source_urls('id2'))
self.assertEqual(mock_error.call_args_list[0][0][0],
"Couldn't load values from json: %s - %s")
self.assertEqual(mock_error.call_args_list[0][0][1], 'id2')
self.assertIsInstance(mock_error.call_args_list[0][0][2], KeyError)
@patch.object(logging, 'error')
def test_metric_source_urls_index_error(self, mock_error, mock_url_read):
""" Test the metric source urls with empty scan response.. """
mock_url_read.side_effect = [PROJECTS, '[]']
self.assertEqual(["http://url/"],
self.__report.metric_source_urls('id2'))
self.assertEqual(mock_error.call_args_list[0][0][0],
"Couldn't load values from json: %s - %s")
self.assertEqual(mock_error.call_args_list[0][0][1], 'id2')
self.assertIsInstance(mock_error.call_args_list[0][0][2], IndexError)
def test_metric_source_urls_on_error(self, mock_url_read):
""" Test the metric source urls when an error occurs. """
mock_url_read.side_effect = [
PROJECTS,
urllib.error.HTTPError(None, None, None, None, None)
]
self.assertEqual(["http://url/"],
self.__report.metric_source_urls('id2'))
self.assertEqual([
call('http://url/CxRestAPI/projects'),
call('http://url/CxRestAPI/sast/scans?projectId=22&last=1')
], mock_url_read.call_args_list)
def test_url(self, mock_url_read):
""" Test the metric source base url. """
mock_url_read.return_value = LAST_SCAN
self.assertEqual("http://url/", self.__report.url())
def test_datetime(self, mock_url_read):
""" Test the date and time of the report. """
mock_url_read.side_effect = [PROJECTS, LAST_SCAN]
self.assertEqual(datetime.datetime(2017, 10, 24, 20, 0, 47),
self.__report.datetime('id2'))
self.assertEqual([
call('http://url/CxRestAPI/projects'),
call('http://url/CxRestAPI/sast/scans?projectId=22&last=1')
], mock_url_read.call_args_list)
def test_datetime_http_error(self, mock_url_read):
""" Test the date and time of the report. """
mock_url_read.side_effect = [
PROJECTS,
urllib.error.HTTPError(None, None, None, None, None)
]
self.assertEqual(datetime.datetime.min, self.__report.datetime('id2'))
self.assertEqual([
call('http://url/CxRestAPI/projects'),
call('http://url/CxRestAPI/sast/scans?projectId=22&last=1')
], mock_url_read.call_args_list)
@patch.object(logging, 'error')
def test_datetime_missing(self, mock_error, mock_url_read):
""" Test a missing date and time of the report. """
mock_url_read.side_effect = [PROJECTS, '[{"id": 202222}]']
self.assertEqual(datetime.datetime.min, self.__report.datetime('id2'))
self.assertEqual(
mock_error.call_args_list[0][0][0],
"Couldn't parse date and time for project %s from %s: %s")
self.assertEqual(mock_error.call_args_list[0][0][1], 'id2')
self.assertEqual(mock_error.call_args_list[0][0][2], 'http://url/')
self.assertIsInstance(mock_error.call_args_list[0][0][3], KeyError)
@patch.object(logging, 'error')
def test_datetime_empty_scan(self, mock_error, mock_url_read):
""" Test a missing scan data. """
mock_url_read.side_effect = [PROJECTS, '[]']
self.assertEqual(datetime.datetime.min, self.__report.datetime('id2'))
self.assertEqual(
mock_error.call_args_list[0][0][0],
"Couldn't parse date and time for project %s from %s: %s")
self.assertEqual(mock_error.call_args_list[0][0][1], 'id2')
self.assertEqual(mock_error.call_args_list[0][0][2], 'http://url/')
self.assertIsInstance(mock_error.call_args_list[0][0][3], IndexError)
@patch.object(logging, 'error')
def test_datetime_format_error(self, mock_error, mock_url_read):
""" Test a invalid date and time of the report. """
mock_url_read.side_effect = [
PROJECTS,
'[{"id": 3, "dateAndTime": {"finishedOn": "2017-40-24T20:00:47.553"}}]'
]
self.assertEqual(datetime.datetime.min, self.__report.datetime('id2'))
self.assertEqual(
mock_error.call_args_list[0][0][0],
"Couldn't parse date and time for project %s from %s: %s")
self.assertEqual(mock_error.call_args_list[0][0][1], 'id2')
self.assertEqual(mock_error.call_args_list[0][0][2], 'http://url/')
self.assertIsInstance(mock_error.call_args_list[0][0][3], ValueError)
@patch.object(logging, 'error')
def test_nr_warnings_on_missing_values(self, mock_error, mock_url_read):
""" Test dealing with empty list of values. """
mock_url_read.side_effect = [PROJECTS, '{}']
self.assertEqual(-1, self.__report.nr_warnings(['id2'], 'medium'))
self.assertEqual(
mock_error.call_args_list[0][0][0],
"Couldn't parse alerts for project %s with %s risk level from %s: %s"
)
self.assertEqual(mock_error.call_args_list[0][0][1], 'id2')
self.assertEqual(mock_error.call_args_list[0][0][2], 'medium')
self.assertEqual(mock_error.call_args_list[0][0][3], 'http://url/')
self.assertIsInstance(mock_error.call_args_list[0][0][4], KeyError)