async def __effort_type_landing_url(self, effort_type: str) -> URL:
"""Generate a landing url for the effort type."""
url = await super( # pylint: disable=bad-super-call
SonarQubeMetricsBaseClass, self)._landing_url(SourceResponses())
component = self._parameter("component")
branch = self._parameter("branch")
return URL(f"{url}/component_measures?id={component}&metric={effort_type}&branch={branch}")
async def __get_commits_recursively(self,
file_path: str,
first_call: bool = True
) -> SourceResponses:
"""Get the commits of files recursively."""
tree_api = await self._gitlab_api_url(
f"repository/tree?path={file_path}&ref={self._parameter('branch', quote=True)}"
)
tree_response = (await super()._get_source_responses(tree_api))[0]
tree = await tree_response.json()
file_paths = [
quote(item["path"], safe="") for item in tree
if item["type"] == "blob"
]
folder_paths = [
quote(item["path"], safe="") for item in tree
if item["type"] == "tree"
]
if not tree and first_call:
file_paths = [file_path]
commits = [self.__last_commit(file_path)
for file_path in file_paths] + [
self.__get_commits_recursively(folder_path,
first_call=False)
for folder_path in folder_paths
]
return SourceResponses(
responses=list(itertools.chain(*(await asyncio.gather(*commits)))))
async def __issue_landing_url(self, issue_key: str) -> URL:
"""Generate a landing url for the issue."""
url = await super()._landing_url(SourceResponses())
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/project/issues?id={component}&issues={issue_key}&open={issue_key}&branch={branch}"
)
async def __hotspot_landing_url(self, hotspot_key: str) -> URL:
"""Generate a landing url for the hotspot."""
url = await SonarQubeCollector._landing_url(self, SourceResponses()) # pylint: disable=protected-access
component = self._parameter("component")
branch = self._parameter("branch")
return URL(
f"{url}/security_hotspots?id={component}&hotspots={hotspot_key}&branch={branch}"
)
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""Override because we want to do a post request to login."""
api_url = urls[0]
credentials = dict(username=self._parameter("username"), password=self._parameter("password"))
response = await self._session.post(f"{api_url}/users/login", data=credentials)
self.__token = (await response.json())["token"]
await self.__get_board()
await self.__get_lists()
await self.__get_cards()
return SourceResponses(responses=[response], api_url=api_url)
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""Override because we need to do a post request and need to separately get the entities."""
api_url = urls[0]
auth = aiohttp.BasicAuth(str(self._parameter("private_token")))
response = await self._session.post(api_url, auth=auth, json=dict(query=self._parameter("wiql")))
ids = [str(work_item["id"]) for work_item in (await response.json()).get("workItems", [])]
if not ids:
return SourceResponses(responses=[response], api_url=api_url)
ids_string = ",".join(ids[: min(self.MAX_IDS_PER_WORK_ITEMS_API_CALL, SourceMeasurement.MAX_ENTITIES)])
work_items_url = URL(f"{await super()._api_url()}/_apis/wit/workitems?ids={ids_string}&api-version=4.1")
work_items = await super()._get_source_responses(work_items_url)
work_items.insert(0, response)
return work_items
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
security_types = self._parameter(self.types_parameter)
vulnerabilities = await super()._parse_source_responses(SourceResponses(responses=[responses[0]])) \
if "vulnerability" in security_types else SourceMeasurement()
if "security_hotspot" in security_types:
json = await responses[-1].json()
nr_hotspots = int(json.get("paging", {}).get("total", 0))
hotspots = [await self.__entity(hotspot) for hotspot in json.get("hotspots", [])]
else:
nr_hotspots = 0
hotspots = []
return SourceMeasurement(
value=str(int(vulnerabilities.value or 0) + nr_hotspots), entities=vulnerabilities.entities + hotspots)
async def __safely_get_source_responses(self) -> SourceResponses:
"""Connect to the source and get the data, without failing. This method should not be overridden
because it makes sure the collection of source data never causes the collector to fail."""
api_url = safe_api_url = self.__class__.__name__
try:
api_url = await self._api_url()
safe_api_url = tokenless(api_url) or self.__class__.__name__
responses = await self._get_source_responses(api_url)
logging.info("Retrieved %s", safe_api_url)
return responses
except aiohttp.ClientError as reason:
error = tokenless(str(reason)) if str(reason) else reason.__class__.__name__
logging.warning("Failed to retrieve %s: %s", safe_api_url, error)
except Exception as reason: # pylint: disable=broad-except
error = stable_traceback(traceback.format_exc())
logging.error("Failed to retrieve %s: %s", safe_api_url, reason)
return SourceResponses(api_url=URL(api_url), connection_error=error)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the selected security types."""
security_types = self._parameter(self.types_parameter)
vulnerabilities = (await super()._parse_source_responses(
SourceResponses(responses=[responses[0]])) if "vulnerability"
in security_types else SourceMeasurement())
if "security_hotspot" in security_types:
json = await responses[-1].json()
hotspots = [
await self.__entity(hotspot)
for hotspot in json.get("hotspots", [])
if hotspot["vulnerabilityProbability"] in
self._review_priorities()
]
else:
hotspots = []
return SourceMeasurement(
value=str(int(vulnerabilities.value or 0) + len(hotspots)),
entities=vulnerabilities.entities + hotspots,
)
str(exception)) if str(exception) else exception.__class__.__name__
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""Open the url(s). Can be overridden if a post request is needed or serial requests need to be made."""
kwargs: dict[str, Any] = {}
credentials = self._basic_auth_credentials()
if credentials is not None:
kwargs["auth"] = aiohttp.BasicAuth(credentials[0], credentials[1])
if headers := self._headers():
kwargs["headers"] = headers
tasks = [self._session.get(url, **kwargs) for url in urls if url]
responses = await asyncio.gather(*tasks, return_exceptions=True)
for response in responses:
if isinstance(response, Exception):
raise response
return SourceResponses(responses=list(responses), api_url=urls[0])
def _basic_auth_credentials(self) -> Optional[tuple[str, str]]:
"""Return the basic authentication credentials, if any."""
if token := cast(str, self.__parameters.get("private_token", "")):
return token, ""
username = cast(str, self.__parameters.get("username", ""))
password = cast(str, self.__parameters.get("password", ""))
return (username, password) if username and password else None
def _headers(self) -> dict[str, str]: # pylint: disable=no-self-use
"""Return the headers for the get request."""
return {}
async def __safely_parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
@staticmethod
def __logsafe_exception(exception: Exception) -> str:
"""Return a log-safe version of the exception."""
return tokenless(
str(exception)) if str(exception) else exception.__class__.__name__
async def _get_source_responses(self, *urls: URL) -> SourceResponses:
"""Open the url(s). Can be overridden if a post request is needed or serial requests need to be made."""
kwargs: Dict[str, Any] = {}
credentials = self._basic_auth_credentials()
if credentials is not None:
kwargs["auth"] = aiohttp.BasicAuth(credentials[0], credentials[1])
if headers := self._headers():
kwargs["headers"] = headers
tasks = [self._session.get(url, **kwargs) for url in urls if url]
return SourceResponses(responses=list(await asyncio.gather(*tasks)),
api_url=urls[0])
def _basic_auth_credentials(self) -> Optional[Tuple[str, str]]:
"""Return the basic authentication credentials, if any."""
if token := cast(str, self.__parameters.get("private_token", "")):
return token, ""
username = cast(str, self.__parameters.get("username", ""))
password = cast(str, self.__parameters.get("password", ""))
return (username, password) if username and password else None
def _headers(self) -> Dict[str, str]: # pylint: disable=no-self-use
"""Return the headers for the get request."""
return {}
async def __safely_parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
