def test_uppercase_hash(self): """Test that an url with uppercase hash is returned without the hash.""" url = URL( "https://test2.app.example.org:1234/main.58064CB8D36474BD79F9.js") expected_url = URL( "https://test2.app.example.org:1234/main.hashremoved.js") self.assertEqual(expected_url, hashless(url))
async def __get_board(self) -> None: """Return the board specified by the user.""" api_url = await self._api_url() user_id = (await self._get_json(URL(f"{api_url}/api/user")))["_id"] boards = await self._get_json(URL(f"{api_url}/api/users/{user_id}/boards")) self._board = [board for board in boards if self._parameter("board") in board.values()][0] self._board_url = f"{api_url}/api/boards/{self._board['_id']}"
async def __get_cards(self) -> None: """Get the cards for the list.""" for lst in self._lists: list_url = f"{self._board_url}/lists/{lst['_id']}" cards = await self._get_json(URL(f"{list_url}/cards")) full_cards = [await self._get_json(URL(f"{list_url}/cards/{card['_id']}")) for card in cards] self._cards[lst["_id"]] = [card for card in full_cards if not self._ignore_card(card)]
def test_hash(self): """Test that an url with hash is returned without the hash.""" url = URL( "https://test1.app.example.org:1234/main.58064cb8d36474bd79f9.js") expected_url = URL( "https://test1.app.example.org:1234/main.hashremoved.js") self.assertEqual(expected_url, hashless(url))
async def _api_url(self) -> URL: url = await super()._api_url() fields_url = URL(f"{url}/rest/api/2/field") response = (await super()._get_source_responses(fields_url))[0] self._field_ids = {field["name"].lower(): field["id"] for field in await response.json()} jql = str(self._parameter("jql", quote=True)) fields = self._fields() return URL(f"{url}/rest/api/2/search?jql={jql}&fields={fields}&maxResults=500")
async def _get_source_responses(self, *urls: URL) -> SourceResponses: """In addition to the suppressed rules, also get issues closed as false positive and won't fix from SonarQube as well as the total number of violations.""" url = await SourceCollector._api_url(self) # pylint: disable=protected-access component = self._parameter("component") branch = self._parameter("branch") all_issues_api_url = URL(f"{url}/api/issues/search?componentKeys={component}&branch={branch}") resolved_issues_api_url = URL( f"{all_issues_api_url}&status=RESOLVED&resolutions=WONTFIX,FALSE-POSITIVE&ps=500&" f"severities={self._violation_severities()}&types={self._violation_types()}") return await super()._get_source_responses(*(urls + (resolved_issues_api_url, all_issues_api_url)))
async def _get_source_responses(self, *urls: URL) -> SourceResponses: api_urls = [] security_types = self._parameter(self.types_parameter) component = self._parameter("component") branch = self._parameter("branch") base_url = await SonarQubeCollector._api_url(self) # pylint: disable=protected-access if "vulnerability" in security_types: api_urls.append( URL(f"{base_url}/api/issues/search?componentKeys={component}&resolved=false&ps=500&" f"severities={self._violation_severities()}&types={self._violation_types()}&branch={branch}")) if "security_hotspot" in security_types: api_urls.append( URL(f"{base_url}/api/hotspots/search?projectKey={component}&status=TO_REVIEW&ps=500&branch={branch}")) return await super()._get_source_responses(*api_urls)
async def __get_lists(self) -> None: """Return the lists on the board.""" self._lists = [ lst for lst in await self._get_json(URL(f"{self._board_url}/lists")) if not self.__ignore_list(lst) ]
async def __effort_type_landing_url(self, effort_type: str) -> URL: """Generate a landing url for the effort type.""" url = await super( # pylint: disable=bad-super-call SonarQubeMetricsBaseClass, self)._landing_url(SourceResponses()) component = self._parameter("component") branch = self._parameter("branch") return URL(f"{url}/component_measures?id={component}&metric={effort_type}&branch={branch}")
async def _landing_url(self, responses: SourceResponses) -> URL: url = await super()._landing_url(responses) component = self._parameter("component") branch = self._parameter("branch") metric = self._landing_url_metric_key() metric_parameter = f"&metric={metric}" if metric else "" return URL(f"{url}/component_measures?id={component}{metric_parameter}&branch={branch}")
async def _api_url(self) -> URL: """Extend to add the jobs API path and parameters.""" url = await super()._api_url() job_attrs = "buildable,color,url,name,builds[result,timestamp]" return URL( f"{url}/api/json?tree=jobs[{job_attrs},jobs[{job_attrs},jobs[{job_attrs}]]]" )
def __init__(self) -> None: self.server_url: Final[URL] = URL( f"http://{os.environ.get('SERVER_HOST', 'localhost')}:{os.environ.get('SERVER_PORT', '5001')}" ) self.data_model: JSON = {} self.last_parameters: Dict[str, Any] = {} self.next_fetch: Dict[str, datetime] = {}
async def _parse_source_responses( self, responses: SourceResponses) -> SourceMeasurement: """Override to parse the sprint values from the responses.""" api_url = await self._api_url() board_id = parse_qs(urlparse(str( responses.api_url)).query)["rapidViewId"][0] entity_url = URL( f"{api_url}/secure/RapidBoard.jspa?rapidView={board_id}&view=reporting&chart=sprintRetrospective&sprint=" ) json = await responses[0].json() # The JSON contains a list with sprints and a dict with the committed and completed points, indexed by sprint id sprints, points = json["sprints"], json["velocityStatEntries"] velocity_type = str(self._parameter("velocity_type")) nr_sprints = int(str(self._parameter("velocity_sprints"))) # Get the points, either completed, committed, or completed minus committed, as determined by the velocity type sprint_values = [ self.__velocity(points[str(sprint["id"])], velocity_type) for sprint in sprints[:nr_sprints] ] entities = Entities( self.__entity(sprint, points[str(sprint["id"])], velocity_type, entity_url) for sprint in sprints) return SourceMeasurement( value=str(round(sum(sprint_values) / len(sprint_values))) if sprint_values else "0", entities=entities)
async def _get_source_responses(self, *urls: URL) -> SourceResponses: """Extend to pass the Greenhopper velocity chart API.""" board_id = await self.__board_id(urls[0]) api_url = URL( f"{urls[0]}/rest/greenhopper/1.0/rapid/charts/velocity.json?rapidViewId={board_id}" ) return await super()._get_source_responses(api_url)
async def _get_source_responses(self, *urls: URL) -> SourceResponses: """Extend to get the scan results.""" await super()._get_source_responses(*urls) # Get token stats_api = URL( f"{await self._api_url()}/cxrestapi/sast/scans/{self._scan_id}/resultsStatistics" ) return await SourceCollector._get_source_responses(self, stats_api) # pylint: disable=protected-access
async def _landing_url(self, responses: SourceResponses) -> URL: """Extend to add the issues path and parameters.""" url = await super()._landing_url(responses) component = self._parameter("component") branch = self._parameter("branch") landing_url = f"{url}/project/issues?id={component}&resolved=false&branch={branch}" return URL(landing_url + self.__rules_url_parameter())
async def _landing_url(self, responses: SourceResponses) -> URL: if not responses: return await super()._landing_url(responses) web_url = (await responses[0].json())["web_url"] branch = self._parameter('branch', quote=True) file_path = self._parameter('file_path', quote=True) return URL(f"{web_url}/blob/{branch}/{file_path}")
async def _parse_source_responses( self, responses: SourceResponses) -> SourceMeasurement: entities: Dict[str, Entity] = {} tag_re = re.compile(r"<[^>]*>") risks = cast(List[str], self._parameter("risks")) for alert in await self.__alerts(responses, risks): ids = [ alert.findtext(id_tag, default="") for id_tag in ("alert", "pluginid", "cweid", "wascid", "sourceid") ] name = alert.findtext("name", default="") description = tag_re.sub("", alert.findtext("desc", default="")) risk = alert.findtext("riskdesc", default="") for alert_instance in alert.findall("./instances/instance"): method = alert_instance.findtext("method", default="") uri = self.__stable( hashless(URL(alert_instance.findtext("uri", default="")))) key = md5_hash(f"{':'.join(ids)}:{method}:{uri}") entities[key] = Entity( key=key, old_key=md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}"), name=name, description=description, uri=uri, location=f"{method} {uri}", risk=risk) return SourceMeasurement(entities=list(entities.values()))
async def _repository_id(self) -> str: """Return the repository id belonging to the repository.""" api_url = str(await super()._api_url()) repository = self._parameter("repository") or urllib.parse.unquote(api_url.rsplit("/", 1)[-1]) repositories_url = URL(f"{api_url}/_apis/git/repositories?api-version=4.1") repositories = (await (await super()._get_source_responses(repositories_url))[0].json())["value"] return str([r for r in repositories if repository in (r["name"], r["id"])][0]["id"])
def __stable(self, url: URL) -> URL: """Return the url without the variable parts.""" reg_exps = self._parameter("variable_url_regexp") stable_url = cast(str, url) for reg_exp in reg_exps: stable_url = re.sub(reg_exp, "variable-part-removed", stable_url) return URL(stable_url)
async def _api_url(self) -> URL: url = await super()._api_url() component = self._parameter("component") branch = self._parameter("branch") return URL( f"{url}/api/project_analyses/search?project={component}&branch={branch}" )
async def _api_url(self) -> URL: url = await super()._api_url() component = self._parameter("component") branch = self._parameter("branch") return URL( f"{url}/api/measures/component?component={component}&metricKeys={self._metric_keys()}&branch={branch}" )
async def _landing_url(self, responses: SourceResponses) -> URL: url = await super()._landing_url(responses) component = self._parameter("component") branch = self._parameter("branch") return URL( f"{url}/component_measures?id={component}&metric=tests&branch={branch}" )
async def __url_with_auth(self, api_part: str) -> URL: """Return the authentication URL parameters.""" sep = "&" if "?" in api_part else "?" api_key = self._parameter("api_key") token = self._parameter("token") return URL( f"{await self._api_url()}/{api_part}{sep}key={api_key}&token={token}" )
async def __issue_landing_url(self, issue_key: str) -> URL: """Generate a landing url for the issue.""" url = await super()._landing_url(SourceResponses()) component = self._parameter("component") branch = self._parameter("branch") return URL( f"{url}/project/issues?id={component}&issues={issue_key}&open={issue_key}&branch={branch}" )
async def _api_url(self) -> URL: url = await super()._api_url() component = self._parameter("component") branch = self._parameter("branch") metric_keys = "tests,test_errors,test_failures,skipped_tests" return URL( f"{url}/api/measures/component?component={component}&metricKeys={metric_keys}&branch={branch}" )
async def __hotspot_landing_url(self, hotspot_key: str) -> URL: """Generate a landing url for the hotspot.""" url = await SonarQubeCollector._landing_url(self, SourceResponses()) # pylint: disable=protected-access component = self._parameter("component") branch = self._parameter("branch") return URL( f"{url}/security_hotspots?id={component}&hotspots={hotspot_key}&branch={branch}" )
async def _api_url(self) -> URL: """Extend to add the project analyses path and parameters.""" url = await super()._api_url() component = self._parameter("component") branch = self._parameter("branch") return URL( f"{url}/api/project_analyses/search?project={component}&branch={branch}" )
async def _gitlab_api_url(self, api: str) -> URL: """Return a GitLab API url with private token, if present in the parameters.""" url = await super()._api_url() project = self._parameter("project", quote=True) api_url = f"{url}/api/v4/projects/{project}" + (f"/{api}" if api else "") sep = "&" if "?" in api_url else "?" api_url += f"{sep}per_page=100" return URL(api_url)
def __alert_instance_entity(self, ids, entity_kwargs, alert_instance) -> Entity: """Create an alert instance entity.""" method = alert_instance.findtext("method", default="") uri = self.__stable_url(hashless(URL(alert_instance.findtext("uri", default="")))) key = md5_hash(f"{':'.join(ids)}:{method}:{uri}") old_key = md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}") location = f"{method} {uri}" return Entity(key=key, old_key=old_key, uri=uri, location=location, **entity_kwargs)