def test_uppercase_hash(self):
"""Test that an url with uppercase hash is returned without the hash."""
url = URL(
"https://test2.app.example.org:1234/main.58064CB8D36474BD79F9.js")
expected_url = URL(
"https://test2.app.example.org:1234/main.hashremoved.js")
self.assertEqual(expected_url, hashless(url))
async def __get_board(self) -> None:
"""Return the board specified by the user."""
api_url = await self._api_url()
user_id = (await self._get_json(URL(f"{api_url}/api/user")))["_id"]
boards = await self._get_json(URL(f"{api_url}/api/users/{user_id}/boards"))
self._board = [board for board in boards if self._parameter("board") in board.values()][0]
self._board_url = f"{api_url}/api/boards/{self._board['_id']}"
async def __get_cards(self) -> None:
"""Get the cards for the list."""
for lst in self._lists:
list_url = f"{self._board_url}/lists/{lst['_id']}"
cards = await self._get_json(URL(f"{list_url}/cards"))
full_cards = [await self._get_json(URL(f"{list_url}/cards/{card['_id']}")) for card in cards]
self._cards[lst["_id"]] = [card for card in full_cards if not self._ignore_card(card)]
def test_hash(self):
"""Test that an url with hash is returned without the hash."""
url = URL(
"https://test1.app.example.org:1234/main.58064cb8d36474bd79f9.js")
expected_url = URL(
"https://test1.app.example.org:1234/main.hashremoved.js")
self.assertEqual(expected_url, hashless(url))
async def _api_url(self) -> URL:
url = await super()._api_url()
fields_url = URL(f"{url}/rest/api/2/field")
response = (await super()._get_source_responses(fields_url))[0]
self._field_ids = {field["name"].lower(): field["id"] for field in await response.json()}
jql = str(self._parameter("jql", quote=True))
fields = self._fields()
return URL(f"{url}/rest/api/2/search?jql={jql}&fields={fields}&maxResults=500")
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""In addition to the suppressed rules, also get issues closed as false positive and won't fix from SonarQube
as well as the total number of violations."""
url = await SourceCollector._api_url(self) # pylint: disable=protected-access
component = self._parameter("component")
branch = self._parameter("branch")
all_issues_api_url = URL(f"{url}/api/issues/search?componentKeys={component}&branch={branch}")
resolved_issues_api_url = URL(
f"{all_issues_api_url}&status=RESOLVED&resolutions=WONTFIX,FALSE-POSITIVE&ps=500&"
f"severities={self._violation_severities()}&types={self._violation_types()}")
return await super()._get_source_responses(*(urls + (resolved_issues_api_url, all_issues_api_url)))
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
api_urls = []
security_types = self._parameter(self.types_parameter)
component = self._parameter("component")
branch = self._parameter("branch")
base_url = await SonarQubeCollector._api_url(self) # pylint: disable=protected-access
if "vulnerability" in security_types:
api_urls.append(
URL(f"{base_url}/api/issues/search?componentKeys={component}&resolved=false&ps=500&"
f"severities={self._violation_severities()}&types={self._violation_types()}&branch={branch}"))
if "security_hotspot" in security_types:
api_urls.append(
URL(f"{base_url}/api/hotspots/search?projectKey={component}&status=TO_REVIEW&ps=500&branch={branch}"))
return await super()._get_source_responses(*api_urls)
async def __get_lists(self) -> None:
"""Return the lists on the board."""
self._lists = [
lst
for lst in await self._get_json(URL(f"{self._board_url}/lists"))
if not self.__ignore_list(lst)
]
async def __effort_type_landing_url(self, effort_type: str) -> URL:
"""Generate a landing url for the effort type."""
url = await super( # pylint: disable=bad-super-call
SonarQubeMetricsBaseClass, self)._landing_url(SourceResponses())
component = self._parameter("component")
branch = self._parameter("branch")
return URL(f"{url}/component_measures?id={component}&metric={effort_type}&branch={branch}")
async def _landing_url(self, responses: SourceResponses) -> URL:
url = await super()._landing_url(responses)
component = self._parameter("component")
branch = self._parameter("branch")
metric = self._landing_url_metric_key()
metric_parameter = f"&metric={metric}" if metric else ""
return URL(f"{url}/component_measures?id={component}{metric_parameter}&branch={branch}")
async def _api_url(self) -> URL:
"""Extend to add the jobs API path and parameters."""
url = await super()._api_url()
job_attrs = "buildable,color,url,name,builds[result,timestamp]"
return URL(
f"{url}/api/json?tree=jobs[{job_attrs},jobs[{job_attrs},jobs[{job_attrs}]]]"
)
def __init__(self) -> None:
self.server_url: Final[URL] = URL(
f"http://{os.environ.get('SERVER_HOST', 'localhost')}:{os.environ.get('SERVER_PORT', '5001')}"
)
self.data_model: JSON = {}
self.last_parameters: Dict[str, Any] = {}
self.next_fetch: Dict[str, datetime] = {}
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the sprint values from the responses."""
api_url = await self._api_url()
board_id = parse_qs(urlparse(str(
responses.api_url)).query)["rapidViewId"][0]
entity_url = URL(
f"{api_url}/secure/RapidBoard.jspa?rapidView={board_id}&view=reporting&chart=sprintRetrospective&sprint="
)
json = await responses[0].json()
# The JSON contains a list with sprints and a dict with the committed and completed points, indexed by sprint id
sprints, points = json["sprints"], json["velocityStatEntries"]
velocity_type = str(self._parameter("velocity_type"))
nr_sprints = int(str(self._parameter("velocity_sprints")))
# Get the points, either completed, committed, or completed minus committed, as determined by the velocity type
sprint_values = [
self.__velocity(points[str(sprint["id"])], velocity_type)
for sprint in sprints[:nr_sprints]
]
entities = Entities(
self.__entity(sprint, points[str(sprint["id"])], velocity_type,
entity_url) for sprint in sprints)
return SourceMeasurement(
value=str(round(sum(sprint_values) /
len(sprint_values))) if sprint_values else "0",
entities=entities)
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""Extend to pass the Greenhopper velocity chart API."""
board_id = await self.__board_id(urls[0])
api_url = URL(
f"{urls[0]}/rest/greenhopper/1.0/rapid/charts/velocity.json?rapidViewId={board_id}"
)
return await super()._get_source_responses(api_url)
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""Extend to get the scan results."""
await super()._get_source_responses(*urls) # Get token
stats_api = URL(
f"{await self._api_url()}/cxrestapi/sast/scans/{self._scan_id}/resultsStatistics"
)
return await SourceCollector._get_source_responses(self, stats_api) # pylint: disable=protected-access
async def _landing_url(self, responses: SourceResponses) -> URL:
"""Extend to add the issues path and parameters."""
url = await super()._landing_url(responses)
component = self._parameter("component")
branch = self._parameter("branch")
landing_url = f"{url}/project/issues?id={component}&resolved=false&branch={branch}"
return URL(landing_url + self.__rules_url_parameter())
async def _landing_url(self, responses: SourceResponses) -> URL:
if not responses:
return await super()._landing_url(responses)
web_url = (await responses[0].json())["web_url"]
branch = self._parameter('branch', quote=True)
file_path = self._parameter('file_path', quote=True)
return URL(f"{web_url}/blob/{branch}/{file_path}")
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
entities: Dict[str, Entity] = {}
tag_re = re.compile(r"<[^>]*>")
risks = cast(List[str], self._parameter("risks"))
for alert in await self.__alerts(responses, risks):
ids = [
alert.findtext(id_tag, default="")
for id_tag in ("alert", "pluginid", "cweid", "wascid",
"sourceid")
]
name = alert.findtext("name", default="")
description = tag_re.sub("", alert.findtext("desc", default=""))
risk = alert.findtext("riskdesc", default="")
for alert_instance in alert.findall("./instances/instance"):
method = alert_instance.findtext("method", default="")
uri = self.__stable(
hashless(URL(alert_instance.findtext("uri", default=""))))
key = md5_hash(f"{':'.join(ids)}:{method}:{uri}")
entities[key] = Entity(
key=key,
old_key=md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}"),
name=name,
description=description,
uri=uri,
location=f"{method} {uri}",
risk=risk)
return SourceMeasurement(entities=list(entities.values()))
async def _repository_id(self) -> str:
"""Return the repository id belonging to the repository."""
api_url = str(await super()._api_url())
repository = self._parameter("repository") or urllib.parse.unquote(api_url.rsplit("/", 1)[-1])
repositories_url = URL(f"{api_url}/_apis/git/repositories?api-version=4.1")
repositories = (await (await super()._get_source_responses(repositories_url))[0].json())["value"]
return str([r for r in repositories if repository in (r["name"], r["id"])][0]["id"])
def __stable(self, url: URL) -> URL:
"""Return the url without the variable parts."""
reg_exps = self._parameter("variable_url_regexp")
stable_url = cast(str, url)
for reg_exp in reg_exps:
stable_url = re.sub(reg_exp, "variable-part-removed", stable_url)
return URL(stable_url)
async def _api_url(self) -> URL:
url = await super()._api_url()
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/api/project_analyses/search?project={component}&branch={branch}"
)
async def _api_url(self) -> URL:
url = await super()._api_url()
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/api/measures/component?component={component}&metricKeys={self._metric_keys()}&branch={branch}"
)
async def _landing_url(self, responses: SourceResponses) -> URL:
url = await super()._landing_url(responses)
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/component_measures?id={component}&metric=tests&branch={branch}"
)
async def __url_with_auth(self, api_part: str) -> URL:
"""Return the authentication URL parameters."""
sep = "&" if "?" in api_part else "?"
api_key = self._parameter("api_key")
token = self._parameter("token")
return URL(
f"{await self._api_url()}/{api_part}{sep}key={api_key}&token={token}"
)
async def __issue_landing_url(self, issue_key: str) -> URL:
"""Generate a landing url for the issue."""
url = await super()._landing_url(SourceResponses())
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/project/issues?id={component}&issues={issue_key}&open={issue_key}&branch={branch}"
)
async def _api_url(self) -> URL:
url = await super()._api_url()
component = self._parameter("component")
branch = self._parameter("branch")
metric_keys = "tests,test_errors,test_failures,skipped_tests"
return URL(
f"{url}/api/measures/component?component={component}&metricKeys={metric_keys}&branch={branch}"
)
async def __hotspot_landing_url(self, hotspot_key: str) -> URL:
"""Generate a landing url for the hotspot."""
url = await SonarQubeCollector._landing_url(self, SourceResponses()) # pylint: disable=protected-access
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/security_hotspots?id={component}&hotspots={hotspot_key}&branch={branch}"
)
async def _api_url(self) -> URL:
"""Extend to add the project analyses path and parameters."""
url = await super()._api_url()
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/api/project_analyses/search?project={component}&branch={branch}"
)
async def _gitlab_api_url(self, api: str) -> URL:
"""Return a GitLab API url with private token, if present in the parameters."""
url = await super()._api_url()
project = self._parameter("project", quote=True)
api_url = f"{url}/api/v4/projects/{project}" + (f"/{api}" if api else "")
sep = "&" if "?" in api_url else "?"
api_url += f"{sep}per_page=100"
return URL(api_url)
def __alert_instance_entity(self, ids, entity_kwargs, alert_instance) -> Entity:
"""Create an alert instance entity."""
method = alert_instance.findtext("method", default="")
uri = self.__stable_url(hashless(URL(alert_instance.findtext("uri", default=""))))
key = md5_hash(f"{':'.join(ids)}:{method}:{uri}")
old_key = md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}")
location = f"{method} {uri}"
return Entity(key=key, old_key=old_key, uri=uri, location=location, **entity_kwargs)
