def test_get_components_show(self):
component_name = SonarAPITestCase.component_name
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
mock_result = {
'component': {
'organization': 'default-organization',
'id': 'AWv059V6TlMPYRjXygTi',
'key': 'seecode-ui',
'name': 'seecode-ui',
'qualifier': 'TRK',
'analysisDate': '2019-07-15T09:12:09+0000',
'tags': [],
'visibility': 'public',
'version': '1.0'
},
'ancestors': []
}
if SonarAPITestCase.enable_mock_env:
sonar.get_components_show = MagicMock(
return_value=mock_result) # fake data
info = sonar.get_components_show(component_name)
self.assertEqual(info['component']['id'], 'AWv059V6TlMPYRjXygTi')
self.assertEqual(info['component']['key'], component_name)
self.assertEqual(info['component']['name'], component_name)
def test_get_qualityprofiles(self):
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
mock_result = [[{
'key': 'AWvzktytTlMPYRjXygTG',
'name': 'Sonar way',
'language': 'yaml',
'languageName': 'YAML',
'isInherited': False,
'isDefault': True,
'activeRuleCount': 19,
'activeDeprecatedRuleCount': 0,
'rulesUpdatedAt': '2019-07-15T03:00:16+0000',
'lastUsed': '2019-07-15T10:30:42+0000',
'organization': 'default-organization',
'isBuiltIn': True,
'actions': {
'edit': False,
'setAsDefault': False,
'copy': True,
'associateProjects': False,
'delete': False
}
}, {
'key': 'AWvzktytTlMPYRjXygTG',
'name': 'Sonar way',
'language': 'yaml',
'languageName': 'YAML',
'isInherited': False,
'isDefault': True,
'activeRuleCount': 19,
'activeDeprecatedRuleCount': 0,
'rulesUpdatedAt': '2019-07-15T03:00:16+0000',
'lastUsed': '2019-07-15T10:30:42+0000',
'organization': 'default-organization',
'isBuiltIn': True,
'actions': {
'edit': False,
'setAsDefault': False,
'copy': True,
'associateProjects': False,
'delete': False
}
}]]
if SonarAPITestCase.enable_mock_env:
sonar.get_qualityprofiles = MagicMock(
return_value=mock_result) # fake data
for qualityprofiles in sonar.get_qualityprofiles():
self.assertTrue(isinstance(qualityprofiles, list))
for _ in qualityprofiles:
self.assertIsNotNone(_['key'])
self.assertIsNotNone(_['name'])
self.assertIsNotNone(_['language'])
self.assertIsNotNone(_['languageName'])
self.assertIsNotNone(_['isDefault'])
self.assertIsNotNone(_['activeRuleCount'])
self.assertIsNotNone(_['activeDeprecatedRuleCount'])
self.assertIsNotNone(_['organization'])
self.assertIsNotNone(_['isBuiltIn'])
def test_get_components_search(self):
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
for components in sonar.get_components_search():
self.assertTrue(isinstance(components, list))
for _ in components:
self.assertIsNotNone(_['organization'])
self.assertIsNotNone(_['id'])
self.assertIsNotNone(_['key'])
self.assertIsNotNone(_['name'])
self.assertIsNotNone(_['qualifier'])
self.assertIsNotNone(_['project'])
def __start_blacklist(self, project):
"""
:param project:
:return:
"""
try:
if not os.path.isfile(self._sonar_scanner_path):
msg = "File path for 'sonar_scanner' not found, sonar_scanner_path: {0}".format(
self._sonar_scanner_path)
raise SonarScannerFailureException(msg)
self.sonarqube = SonarAPIHandler(
token=self._api_token,
logger=project.logger,
sonar_server=self._api_domain,
http_timeout_retry=self._http_timeout_retry,
http_failed_retry=self._http_failed_retry,
http_timeout=self._http_timeout,
)
self.__make_sonar_config(project)
# 开始执行 sonar-scanner 命令
exec_cmd(' '.join([
'cd', project.scan_path, '&&', self._sonar_scanner_path,
'1>>{0} 2>&1 &'.format(project.log_file)
]))
if self._grep.exec_match(cmd='"EXECUTION SUCCESS" {0}'.format(
project.log_file),
match_str='EXECUTION SUCCESS'):
self.__sync_issues(project)
else:
reason = self.__get_failed_reason(project)
project.update_scan_message(title='EXECUTION FAILURE!',
reason=reason,
level=2)
raise SonarScannerFailureException(
'[SonarScanner] EXECUTION FAILURE! {0}'.format(reason))
except (CodeBaseException, gevent.exceptions.LoopExit) as ex:
project.logger.error(ex)
raise ex
def test_get_languages(self):
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
mock_result = [[{
'key': 'cs',
'name': 'C#'
}, {
'key': 'flex',
'name': 'Flex'
}, {
'key': 'java',
'name': 'Java'
}]]
if SonarAPITestCase.enable_mock_env:
sonar.get_languages = MagicMock(
return_value=mock_result) # fake data
for languages in sonar.get_languages():
self.assertTrue(isinstance(languages, list))
for _ in languages:
self.assertIsNotNone(_['key'])
self.assertIsNotNone(_['name'])
def test_get_ce_component(self):
component_name = SonarAPITestCase.component_name
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
mock_result = {
'queue': [],
'current': {
'id': 'AWv059V6TlMPYRjXygTo',
'type': 'REPORT',
'componentId': 'AWv059V6TlMPYRjXygTi',
'componentKey': 'seecode-ui',
'componentName': 'seecode-ui',
'componentQualifier': 'TRK',
'analysisId': 'AWv059m5kx-ffhfhf55z',
'status': 'SUCCESS',
'submittedAt': '2019-07-15T09:12:42+0000',
'submitterLogin': '******',
'startedAt': '2019-07-15T09:12:43+0000',
'executedAt': '2019-07-15T09:12:50+0000',
'executionTimeMs': 7426,
'logs': False,
'hasScannerContext': True,
'organization': 'default-organization'
}
}
with self.assertRaises(MissingImportantScanParameters) as _:
sonar.get_ce_component()
# test component_id
if SonarAPITestCase.enable_mock_env:
sonar.get_ce_component = MagicMock(
return_value=mock_result) # fake data
info = sonar.get_ce_component(component_id='AWv059V6TlMPYRjXygTi')
self.assertEqual(info['current']['componentId'],
'AWv059V6TlMPYRjXygTi')
self.assertEqual(info['current']['componentKey'], component_name)
self.assertEqual(info['current']['componentName'], component_name)
self.assertEqual(info['current']['status'], 'SUCCESS')
# test component_key
if SonarAPITestCase.enable_mock_env:
sonar.get_ce_component = MagicMock(
return_value=mock_result) # fake data
info = sonar.get_ce_component(component_key=component_name)
self.assertEqual(info['current']['componentId'],
'AWv059V6TlMPYRjXygTi')
self.assertEqual(info['current']['componentKey'], component_name)
self.assertEqual(info['current']['componentName'], component_name)
self.assertEqual(info['current']['status'], 'SUCCESS')
class SonarScanner(BaseScanner):
def __init__(self, **kwargs):
"""
"""
super().__init__(**kwargs)
self.component_rule = {}
self.whitelist_rule = {}
self.blacklist_rule = {}
self._grep = GrepCMD()
self._key = 1
self.sonarqube = None
self.pool = ThreadPool(20)
def load_component(self, items):
pass
def load_blacklist(self, items):
pass
def load_whitelist(self, items):
pass
def start_component(self, project):
pass
def start_whitelist(self, project):
pass
def start_blacklist(self, project):
"""
:param project:
:return:
"""
project.logger.info(
"[SonarScanner] Start the start_blacklist method...")
try:
func_timeout(self._engine_timeout,
self.__start_blacklist,
args=(project, ))
except FunctionTimedOut as te:
self.__kill_sonar_scanner_pid(project)
msg = '[SonarScanner] sonar-scanner execution timeout, default timeout: {0}s.'.format(
self._engine_timeout)
project.update_scan_message(title='扫描超时', reason=msg, level=2)
raise EngineExecutionTimeoutException(msg)
project.logger.info("[SonarScanner] Sonar blacklist scan completed.")
def __start_blacklist(self, project):
"""
:param project:
:return:
"""
try:
if not os.path.isfile(self._sonar_scanner_path):
msg = "File path for 'sonar_scanner' not found, sonar_scanner_path: {0}".format(
self._sonar_scanner_path)
raise SonarScannerFailureException(msg)
self.sonarqube = SonarAPIHandler(
token=self._api_token,
logger=project.logger,
sonar_server=self._api_domain,
http_timeout_retry=self._http_timeout_retry,
http_failed_retry=self._http_failed_retry,
http_timeout=self._http_timeout,
)
self.__make_sonar_config(project)
# 开始执行 sonar-scanner 命令
exec_cmd(' '.join([
'cd', project.scan_path, '&&', self._sonar_scanner_path,
'1>>{0} 2>&1 &'.format(project.log_file)
]))
if self._grep.exec_match(cmd='"EXECUTION SUCCESS" {0}'.format(
project.log_file),
match_str='EXECUTION SUCCESS'):
self.__sync_issues(project)
else:
reason = self.__get_failed_reason(project)
project.update_scan_message(title='EXECUTION FAILURE!',
reason=reason,
level=2)
raise SonarScannerFailureException(
'[SonarScanner] EXECUTION FAILURE! {0}'.format(reason))
except (CodeBaseException, gevent.exceptions.LoopExit) as ex:
project.logger.error(ex)
raise ex
def __kill_sonar_scanner_pid(self, project):
"""
:param project:
:return:
"""
try:
project.logger.info(
'[SonarScanner] Get the pid number of the sonar-scanner ...')
get_sonar_scanner_cmd = 'ps -ef | grep "cd ' + project.scan_path + ' \&\& ' + self._sonar_scanner_path + '"|grep -v grep|awk \'{print($2)}\''
project.logger.debug(
'[SonarScanner] Query pid command: [{0}]'.format(
get_sonar_scanner_cmd))
pid, _ = exec_cmd(get_sonar_scanner_cmd)
pid = parse_int_or_str(pid) or ''
project.logger.debug('[SonarScanner] Get PID: [{0}]'.format(pid))
if pid:
sub_pid_cmd = 'ps -ef | grep "Dproject.home=' + project.scan_path + '"|grep -v grep|awk \'{print($2)}\''
sub_pid, _ = exec_cmd(sub_pid_cmd)
sub_pid = strip(sub_pid)
if '\n' in sub_pid.decode("utf-8"):
sub_pid = [
parse_int_or_str(_p)
for _p in sub_pid.decode("utf-8").split("\n") if _p
]
else:
sub_pid = parse_int_or_str(sub_pid) or ''
kill_sonar_scanner_cmd = 'kill -9 {0} {1}'.format(pid, sub_pid)
status, _ = exec_cmd(kill_sonar_scanner_cmd)
if _:
project.logger.warning(
'[SonarScanner] The termination of the sonar-scanner '
'process failed with the command: [{0}]'.format(
kill_sonar_scanner_cmd))
else:
project.logger.info(
'[SonarScanner] Process [{0},{1}] is terminated.'.
format(pid, sub_pid))
except:
pass
def __get_failed_reason(self, project):
"""
:return:
"""
reason_msg = ''
failed_msg = [
'Failed to upload report',
'SonarQube analysis without pushing the results to the SonarQube server.',
]
for i in failed_msg:
if self._grep.exec_match(cmd='"{0}" {1}'.format(
i, project.log_file),
match_str=i):
return ScanFailedException(
'SonarQube 上传分析报告失败，请联系 SonarQube 管理员。')
return reason_msg
def __make_sonar_config(self, project):
"""
:param project:
:return:
"""
sonar_conf_path = os.path.join(project.scan_path,
'sonar-project.properties')
if not os.path.isfile(sonar_conf_path):
scan_config = self._sonar_project_properties or SONAR_PROJECT_PROPERTIES
scan_config = scan_config.replace('{{project_key}}', project.key)
scan_config = scan_config.replace('{{project_name}}', project.name)
scan_config = scan_config.replace('{{sonar_host}}',
'{0}'.format(self._api_domain))
scan_config = scan_config.replace('{{sonar_login}}',
self._api_token)
with open(sonar_conf_path, 'w') as fp:
fp.write(scan_config)
msg = "[SonarScanner] Didn't find the 'sonar-project.properties' file, " \
"create [{0}] using the default template.".format(sonar_conf_path)
project.logger.warning(msg)
project.update_scan_message(
title="[SonarScanner] 没有找到 'sonar-project.properties'文件",
reason=msg,
level=3)
def __sync_issues(self, project):
"""
同步 sonar 的 issue
:return:
"""
project.logger.info('[SonarScanner] Start sync sonar issues...')
try:
for issue_list in self.sonarqube.get_issues(
project_key=project.key):
for issue in issue_list:
self.__create_issue(project, issue)
# self.pool.spawn(self.__create_issue, project, issue) #
# gevent.wait()
except Exception as ex:
project.logger.error(ex)
def __create_issue(self, project, issue):
"""
:param project:
:param issue:
:return:
"""
try:
if not issue and not isinstance(issue, dict):
return
start_line, end_line = 1, 1
if 'textRange' in issue and ('startLine' in issue['textRange']
or 'endLine' in issue['textRange']):
start_line = issue['textRange']['startLine']
end_line = issue['textRange']['endLine']
_, filename = issue['component'].split(':')
author, author_email = get_git_author(
project.get_last_author(filename))
report_detail_url = '{0}component?id={1}&line={2}'.format(
self._api_domain, issue['component'], start_line)
code_segment = get_line_content(
file=os.path.join(project.scan_path, filename),
start_line=start_line +
int(project.evidence_start_line_offset),
count=project.evidence_count)
vuln = Vulnerability(
task_id=project.task_id,
risk_id=get_risk_by_severity(issue['severity']),
rule_key=issue['rule'],
category=issue['type'],
title=issue['message'],
file=filename,
author=author,
author_email=author_email,
hash=project.get_last_commit(),
start_line=start_line,
end_line=end_line,
report=report_detail_url,
code_example=''.join(code_segment),
engine=self.key,
)
vuln_key = hash_md5('{0}_{1}_{2}'.format(filename, issue['rule'],
start_line))
if vuln_key not in kb.result[project.key]:
kb.result[project.key][vuln_key] = vuln.info
except Exception as ex:
import traceback
traceback.print_exc()
project.logger.error(ex)
def test_get_rules(self):
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
mock_result = [[{
'key':
'yaml',
'name':
'YAML Analyzer',
'filename':
'sonar-yaml-plugin-1.4.3.jar',
'sonarLintSupported':
False,
'hash':
'afe7a59a688b470eeec11ef723aba36e',
'updatedAt':
1563159599961,
'description':
'YAML 1.1 plugin for SonarQube',
'version':
'1.4.3',
'license':
'Apache License, Version 2.0',
'editionBundled':
False,
'homepageUrl':
'https://github.com/sbaudoin/sonar-yaml',
'issueTrackerUrl':
'https://github.com/sbaudoin/sonar-yaml/issues'
}, {
'key':
'python',
'name':
'SonarPython',
'filename':
'sonar-python-plugin-1.14.1.3143.jar',
'sonarLintSupported':
True,
'hash':
'5246ba7166e8e9ab17d08a6514e9a086',
'updatedAt':
1563159599961,
'description':
'Code Analyzer for Python',
'version':
'1.14.1 (build 3143)',
'license':
'GNU LGPL 3',
'organizationName':
'SonarSource and Waleri Enns',
'editionBundled':
False,
'homepageUrl':
'http://redirect.sonarsource.com/plugins/python.html',
'issueTrackerUrl':
'https://jira.sonarsource.com/browse/SONARPY',
'implementationBuild':
'eed7b315b6116fe462a19c771013bf3891c92a97'
}]]
if SonarAPITestCase.enable_mock_env:
sonar.get_plugins = MagicMock(
return_value=mock_result) # fake data
for plugins in sonar.get_plugins():
self.assertTrue(isinstance(plugins, list))
for _ in plugins:
self.assertIsNotNone(_['key'])
self.assertIsNotNone(_['name'])
self.assertIsNotNone(_['filename'])
self.assertIsNotNone(_['hash'])
self.assertIsNotNone(_['version'])
def test_validate_authentication(self):
self.sonar_server = os.getenv("SONAR_API_DOMAIN")
with self.assertRaises(SonarQubeAuthenticationFailed) as _:
SonarAPIHandler(token="1234567890",
sonar_server=SonarAPITestCase.sonar_server)
def test_get_rule_info(self):
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
rule_key = 'squid:S2095'
mock_result = {
'rule': {
'key':
'squid:S2095',
'repo':
'squid',
'name':
'Resources should be closed',
'createdAt':
'2019-07-15T02:26:14+0000',
'htmlDesc':
'',
'severity':
'BLOCKER',
'status':
'READY',
'isTemplate':
False,
'tags': [],
'sysTags': ['cert', 'cwe', 'denial-of-service', 'leak'],
'lang':
'java',
'langName':
'Java',
'params': [{
'key': 'excludedResourceTypes',
'htmlDesc':
'Comma separated list of the excluded resource types, using fully qualified names (example: "org.apache.hadoop.fs.FileSystem")',
'type': 'STRING'
}],
'defaultDebtRemFnType':
'CONSTANT_ISSUE',
'defaultDebtRemFnOffset':
'5min',
'debtOverloaded':
False,
'debtRemFnType':
'CONSTANT_ISSUE',
'debtRemFnOffset':
'5min',
'defaultRemFnType':
'CONSTANT_ISSUE',
'defaultRemFnBaseEffort':
'5min',
'remFnType':
'CONSTANT_ISSUE',
'remFnBaseEffort':
'5min',
'remFnOverloaded':
False,
'scope':
'MAIN',
'type':
'BUG'
},
'actives': []
}
if SonarAPITestCase.enable_mock_env:
sonar.get_rule_info = MagicMock(
return_value=mock_result) # fake data
info = sonar.get_rule_info(rule_key=rule_key)
self.assertEqual(info['rule']['key'], rule_key)
self.assertEqual(info['rule']['repo'], 'squid')
self.assertEqual(info['rule']['name'], 'Resources should be closed')
self.assertEqual(info['rule']['type'], 'BUG')
self.assertEqual(info['rule']['severity'], 'BLOCKER')
def test_get_issues(self):
sonar = SonarAPIHandler(token=SonarAPITestCase.token,
sonar_server=SonarAPITestCase.sonar_server)
component_name = SonarAPITestCase.component_name
mock_result = [[{
'key': 'AWv0vLi5kx-ffhfhf3tm',
'rule': 'yaml:CommentsCheck',
'severity': 'INFO',
'component':
'seven_api:source/seven-api-provider/src/main/resources/application-online.yml',
'project': 'seven_api',
'line': 110,
'hash': '98c65da2701d0c636db749f205c0348b',
'textRange': {
'startLine': 110,
'endLine': 110,
'startOffset': 0,
'endOffset': 111
},
'flows': [],
'status': 'OPEN',
'message': 'too few spaces before comment (comments)',
'effort': '2min',
'debt': '2min',
'author': '',
'tags': ['convention'],
'creationDate': '2019-07-15T08:24:47+0000',
'updateDate': '2019-07-15T08:24:47+0000',
'type': 'CODE_SMELL',
'organization': 'default-organization'
}, {
'key': 'AWv0vLi4kx-ffhfhf3tN',
'rule': 'yaml:EmptyLinesCheck',
'severity': 'INFO',
'component':
'seven_api:source/seven-api-provider/src/main/resources/application-online.yml',
'project': 'seven_api',
'line': 158,
'textRange': {
'startLine': 158,
'endLine': 158,
'startOffset': 0,
'endOffset': 0
},
'flows': [],
'status': 'OPEN',
'message': 'too many blank lines (1 > 0) (empty-lines)',
'effort': '2min',
'debt': '2min',
'author': '',
'tags': ['convention'],
'creationDate': '2019-07-15T08:24:47+0000',
'updateDate': '2019-07-15T08:24:47+0000',
'type': 'CODE_SMELL',
'organization': 'default-organization'
}]]
if SonarAPITestCase.enable_mock_env:
sonar.get_rule_info = MagicMock(
return_value=mock_result) # fake data
for issues in sonar.get_issues(project_key=component_name):
self.assertTrue(isinstance(issues, list))
for _ in issues:
self.assertIsNotNone(_['key'])
self.assertIsNotNone(_['rule'])
self.assertIsNotNone(_['severity'])
self.assertIsNotNone(_['component'])
self.assertIsNotNone(_['project'])
self.assertIsNotNone(_['line'])
self.assertIsNotNone(_['textRange'])
self.assertIsNotNone(_['status'])
self.assertIsNotNone(_['message'])
self.assertIsNotNone(_['type'])