async def _parse_source_responses( self, responses: SourceResponses) -> SourceMeasurement: count = 0 total = 0 entities = [] test_results = cast(List[str], self._parameter("test_result")) all_test_results = self._data_model["sources"][ self.source_type]["parameters"]["test_result"]["values"] for response in responses: tree = await parse_source_response_xml(response) stats = tree.findall("statistics/total/stat")[1] for test_result in all_test_results: total += int(stats.get(test_result, 0)) if test_result in test_results: count += int(stats.get(test_result, 0)) for test in tree.findall( f".//test/status[@status='{test_result.upper()}']/.." ): entities.append( Entity(key=test.get("id", ""), name=test.get("name", ""), test_result=test_result)) return SourceMeasurement(value=str(count), total=str(total), entities=entities)
async def _parse_source_responses( self, responses: SourceResponses) -> SourceMeasurement: entities = [] severities = self._parameter("severities") for response in responses: json = await response.json(content_type=None) vulnerabilities = json.get("vulnerabilities", []) if isinstance( json, dict) else [] for vulnerability in vulnerabilities: if vulnerability["severity"].lower() not in severities: continue package_include = " ➜ ".join([str(package) for package in vulnerability["from"][1:]]) \ if isinstance(vulnerability["from"], list) else vulnerability["from"] fix = ", ".join([str(package) for package in vulnerability["fixedIn"]]) \ if isinstance(vulnerability["fixedIn"], list) else vulnerability["fixedIn"] key = md5_hash(f'{vulnerability["id"]}:{package_include}') entities.append( Entity(key=key, cve=vulnerability["title"], package=vulnerability["packageName"], severity=vulnerability["severity"], version=vulnerability['version'], package_include=package_include, fix=fix, url=f"https://snyk.io/vuln/{vulnerability['id']}")) return SourceMeasurement(entities=entities)
async def _parse_source_responses( self, responses: SourceResponses) -> SourceMeasurement: entities: Dict[str, Entity] = {} tag_re = re.compile(r"<[^>]*>") risks = cast(List[str], self._parameter("risks")) for alert in await self.__alerts(responses, risks): ids = [ alert.findtext(id_tag, default="") for id_tag in ("alert", "pluginid", "cweid", "wascid", "sourceid") ] name = alert.findtext("name", default="") description = tag_re.sub("", alert.findtext("desc", default="")) risk = alert.findtext("riskdesc", default="") for alert_instance in alert.findall("./instances/instance"): method = alert_instance.findtext("method", default="") uri = self.__stable( hashless(URL(alert_instance.findtext("uri", default="")))) key = md5_hash(f"{':'.join(ids)}:{method}:{uri}") entities[key] = Entity( key=key, old_key=md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}"), name=name, description=description, uri=uri, location=f"{method} {uri}", risk=risk) return SourceMeasurement(entities=list(entities.values()))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: entities = [ Entity( key=job["name"], name=job["name"], url=job["url"], build_status=self._build_status(job), build_date=str(self._build_datetime(job).date()) if self._build_datetime(job) > datetime.min else "") for job in self.__jobs((await responses[0].json())["jobs"])] return SourceMeasurement(entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: test_results = cast(List[str], self._parameter("test_result")) test_run_names_to_include = cast(List[str], self._parameter("test_run_names_to_include")) or ["all"] test_run_states_to_include = [ value.lower() for value in self._parameter("test_run_states_to_include")] or ["all"] runs = (await responses[0].json()).get("value", []) highest_build: Dict[str, TestRun] = defaultdict(TestRun) for run in runs: name = run.get("name", "Unknown test run name") if test_run_names_to_include != ["all"] and \ not match_string_or_regular_expression(name, test_run_names_to_include): continue state = run.get("state", "Unknown test run state") if test_run_states_to_include != ["all"] and state.lower() not in test_run_states_to_include: continue build_nr = int(run.get("build", {}).get("id", -1)) if build_nr < highest_build[name].build_nr: continue if build_nr > highest_build[name].build_nr: highest_build[name] = TestRun(build_nr) counted_tests = sum(run.get(test_result, 0) for test_result in test_results) highest_build[name].test_count += counted_tests highest_build[name].total_test_count += run.get("totalTests", 0) highest_build[name].entities.append( Entity( key=run["id"], name=name, state=state, build_id=str(build_nr), url=run.get("webAccessUrl", ""), started_date=run.get("startedDate", ""), completed_date=run.get("completedDate", ""), counted_tests=str(counted_tests), incomplete_tests=str(run.get("incompleteTests", 0)), not_applicable_tests=str(run.get("notApplicableTests", 0)), passed_tests=str(run.get("passedTests", 0)), unanalyzed_tests=str(run.get("unanalyzedTests", 0)), total_tests=str(run.get("totalTests", 0)))) test_count = sum(build.test_count for build in highest_build.values()) total_test_count = sum(build.total_test_count for build in highest_build.values()) test_runs = list(itertools.chain.from_iterable([build.entities for build in highest_build.values()])) return SourceMeasurement(value=str(test_count), total=str(total_test_count), entities=test_runs)
def __entity(case_node, case_result: str) -> Entity: """Transform a test case into a test case entity.""" name = case_node.get("name", "<nameless test case>") return Entity(key=name, name=name, class_name=case_node.get("classname", ""), test_result=case_result)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: entities = [ Entity( key=branch["name"], name=branch["name"], commit_date=str(self._commit_datetime(branch).date()), url=str(self._branch_landing_url(branch))) for branch in await self._unmerged_branches(responses)] return SourceMeasurement(entities=entities)
def setUp(self): """Prepare the security warnings metric and sources.""" super().setUp() self.sources = dict( source_id=dict(type="snyk", parameters=dict(url="snyk.json"))) self.metric = dict(type="security_warnings", sources=self.sources, addition="sum") self.direct_dependency = "[email protected]" self.direct_dependency_key = Entity.safe_entity_key( self.direct_dependency) self.direct_dependency_path = [ "package.json@*", self.direct_dependency ] self.vulnerabilities_json = dict(vulnerabilities=[{ "id": "SNYK-JS-ACORN-559469", "severity": "low", "from": self.direct_dependency_path + ["[email protected]", "[email protected]"], }]) self.expected_entity = dict( key=self.direct_dependency_key, dependency=self.direct_dependency, nr_vulnerabilities=1, example_vulnerability="SNYK-JS-ACORN-559469", url="https://snyk.io/vuln/SNYK-JS-ACORN-559469", example_path= "package.json@* ➜ [email protected] ➜ [email protected] ➜ [email protected]", highest_severity="low", )
def entity( # pylint: disable=too-many-arguments self, component: str, entity_type: str, severity: str = None, resolution: str = None, vulnerability_probability: str = None, creation_date: str = None, update_date: str = None) -> Entity: """Create an entity.""" url = self.hotspot_landing_url.format(component) if entity_type == "security_hotspot" else \ self.issue_landing_url.format(component) entity = Entity(key=component, component=component, message=component, type=entity_type, url=url) if severity is not None: entity["severity"] = severity if resolution is not None: entity["resolution"] = resolution if vulnerability_probability is not None: entity["vulnerability_probability"] = vulnerability_probability entity["creation_date"] = creation_date entity["update_date"] = update_date return entity
class SnykSecurityWarnings(JSONFileSourceCollector): """Snyk collector for security warnings.""" async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: """Parse the direct dependencies with vulnerabilities from the responses.""" selected_severities = self._parameter("severities") severities: Dict[str, Set[Severity]] = {} nr_vulnerabilities: Dict[str, int] = {} example_vulnerability = {} for response in responses: json = await response.json(content_type=None) vulnerabilities = json.get("vulnerabilities", []) for vulnerability in vulnerabilities: if (severity := vulnerability["severity"]) not in selected_severities: continue dependency = vulnerability["from"][1] if len(vulnerability["from"]) > 1 else vulnerability["from"][0] severities.setdefault(dependency, set()).add(severity) nr_vulnerabilities[dependency] = nr_vulnerabilities.get(dependency, 0) + 1 path = " ➜ ".join(str(dependency) for dependency in vulnerability["from"]) example_vulnerability[dependency] = (vulnerability["id"], path) entities = [] for dependency in severities: entities.append( Entity( key=dependency, dependency=dependency, nr_vulnerabilities=nr_vulnerabilities[dependency], example_vulnerability=example_vulnerability[dependency][0], url=f"https://snyk.io/vuln/{example_vulnerability[dependency][0]}", example_path=example_vulnerability[dependency][1], highest_severity=self.__highest_severity(severities[dependency]))) return SourceMeasurement(entities=entities)
def entity( # pylint: disable=too-many-arguments component: str, entity_type: str, severity: str = None, resolution: str = None, review_priority: str = None, creation_date: str = None, update_date: str = None, ) -> Entity: """Create an entity.""" url = ( f"https://sonarqube/security_hotspots?id=id&hotspots={component}&branch=master" if entity_type == "security_hotspot" else f"https://sonarqube/project/issues?id=id&issues={component}&open={component}&branch=master" ) entity = Entity( key=component, component=component, message=component, type=entity_type, url=url, creation_date=creation_date, update_date=update_date, ) if severity is not None: entity["severity"] = severity if resolution is not None: entity["resolution"] = resolution if review_priority is not None: entity["review_priority"] = review_priority return entity
def __violation(self, violation: Element, namespaces: Namespaces, models: ModelFilePaths, severities: list[str]) -> Optional[Entity]: """Return the violation as entity.""" location = violation.find("./ns:location", namespaces) if not location: raise SourceCollectorException( f"OJAudit violation {violation} has no location element") severity = violation.findtext("./ns:values/ns:value", default="", namespaces=namespaces) if severities and severity not in severities: return None message = violation.findtext("ns:message", default="", namespaces=namespaces) line_number = violation.findtext(".//ns:line-number", namespaces=namespaces) column_offset = violation.findtext(".//ns:column-offset", namespaces=namespaces) model = models[location.get("model", "")] component = f"{model}:{line_number}:{column_offset}" key = sha1_hash(f"{message}:{component}") entity = Entity(key=key, severity=severity, message=message, component=component) if entity["key"] in self.violation_counts: self.violation_counts[entity["key"]] += 1 return None # Ignore duplicate violation self.violation_counts[entity["key"]] = 1 return entity
async def _parse_entities(self, responses: SourceResponses) -> Entities: """Override to parse the violations.""" entity_attributes = [] for response in responses: json = await response.json(content_type=None) url = json["url"] for violation in json.get("violations", []): for node in violation.get("nodes", []): tags = violation.get("tags", []) impact = node.get("impact") if self.__include_violation(impact, tags): entity_attributes.append( dict( description=violation.get("description"), element=node.get("html"), help=violation.get("helpUrl"), impact=impact, page=url, url=url, tags=", ".join(sorted(tags)), violation_type=violation.get("id"), )) return Entities( Entity(key=self.__create_key(attributes), **attributes) for attributes in entity_attributes)
async def _parse_source_responses( self, responses: SourceResponses) -> SourceMeasurement: impact_levels = self._parameter("impact") entity_attributes = [] for response in responses: json = await response.json(content_type=None) url = json["url"] for violation in json.get("violations", []): for node in violation.get("nodes", []): if node.get("impact") not in impact_levels: continue entity_attributes.append( dict(description=violation.get("description"), element=node.get("html"), help=violation.get("helpUrl"), impact=node.get("impact"), page=url, url=url, violation_type=violation.get("id"))) entities = [ Entity(key=md5_hash(",".join( str(value) for value in attributes.values())), **attributes) for attributes in entity_attributes ] return SourceMeasurement(entities=entities)
class JiraIssues(SourceCollector): """Jira collector for issues.""" SPRINT_NAME_RE = re.compile(r",name=(.*),startDate=") def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._field_ids = {} async def _api_url(self) -> URL: """Extend to get the fields from Jira and create a field name to field id mapping.""" url = await super()._api_url() fields_url = URL(f"{url}/rest/api/2/field") response = (await super()._get_source_responses(fields_url))[0] self._field_ids = {field["name"].lower(): field["id"] for field in await response.json()} jql = str(self._parameter("jql", quote=True)) fields = self._fields() return URL(f"{url}/rest/api/2/search?jql={jql}&fields={fields}&maxResults=500") async def _landing_url(self, responses: SourceResponses) -> URL: """Extend to add the JQL query to the landing URL.""" url = await super()._landing_url(responses) jql = str(self._parameter("jql", quote=True)) return URL(f"{url}/issues/?jql={jql}") def _parameter(self, parameter_key: str, quote: bool = False) -> Union[str, List[str]]: """Extend to replace field names with field ids, if the parameter is a field.""" parameter_value = super()._parameter(parameter_key, quote) if parameter_key.endswith("field"): parameter_value = self._field_ids.get(str(parameter_value).lower(), parameter_value) return parameter_value async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: """Override to get the issues from the responses.""" url = URL(str(self._parameter("url"))) json = await responses[0].json() entities = [self._create_entity(issue, url) for issue in json.get("issues", []) if self._include_issue(issue)] return SourceMeasurement(value=self._compute_value(entities), entities=entities) @classmethod def _compute_value(cls, entities: List[Entity]) -> Value: # pylint: disable=unused-argument """Allow subclasses to compute the value from the entities.""" return None def _create_entity(self, issue: Dict, url: URL) -> Entity: # pylint: disable=no-self-use """Create an entity from a Jira issue.""" fields = issue["fields"] entity_attributes = dict( created=fields["created"], priority=fields.get("priority", {}).get("name"), status=fields.get("status", {}).get("name"), summary=fields["summary"], type=fields.get("issuetype", {}).get("name", "Unknown issue type"), updated=fields.get("updated"), url=f"{url}/browse/{issue['key']}", ) if sprint_field_id := self._field_ids.get("sprint"): entity_attributes["sprint"] = self.__get_sprint_names(fields.get(sprint_field_id) or []) return Entity(key=issue["id"], **entity_attributes)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: jobs = await self.__jobs(responses) entities = [ Entity( key=job["id"], name=job["name"], url=job["web_url"], build_status=job["status"], branch=job["ref"], stage=job["stage"], build_date=str(parse(job["created_at"]).date())) for job in jobs] return SourceMeasurement(entities=entities)
async def _entities(self, metrics: Dict[str, str]) -> List[Entity]: if self._value_key() == "ncloc": # Our user picked non-commented lines of code (ncloc), so we can show the ncloc per language, skipping # languages the user wants to ignore return [ Entity(key=language, language=self.LANGUAGES.get(language, language), ncloc=ncloc) for language, ncloc in self.__language_ncloc(metrics)] return await super()._entities(metrics)
async def __entity(self, hotspot) -> Entity: return Entity( key=hotspot["key"], component=hotspot["component"], message=hotspot["message"], type="security_hotspot", url=await self.__hotspot_landing_url(hotspot["key"]), vulnerability_probability=hotspot["vulnerabilityProbability"].lower())
def __card_to_entity(card, lists) -> Entity: """Convert a card into a entity.""" return Entity(key=card["id"], title=card["name"], url=card["url"], list=lists[card["idList"]], due_date=card["due"], date_last_activity=card["dateLastActivity"])
def __alert_instance_entity(self, ids, entity_kwargs, alert_instance) -> Entity: """Create an alert instance entity.""" method = alert_instance.findtext("method", default="") uri = self.__stable_url(hashless(URL(alert_instance.findtext("uri", default="")))) key = md5_hash(f"{':'.join(ids)}:{method}:{uri}") old_key = md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}") location = f"{method} {uri}" return Entity(key=key, old_key=old_key, uri=uri, location=location, **entity_kwargs)
async def _parse_entities(self, responses: SourceResponses) -> Entities: """Override to get the unmerged branches from the unmerged branches method that subclasses should implement.""" return Entities( Entity( key=branch["name"], name=branch["name"], commit_date=str(self._commit_datetime(branch).date()), url=str(self._branch_landing_url(branch)), ) for branch in await self._unmerged_branches(responses))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: value = str(len((await responses[0].json())["workItems"])) entities = [ Entity( key=work_item["id"], project=work_item["fields"]["System.TeamProject"], title=work_item["fields"]["System.Title"], work_item_type=work_item["fields"]["System.WorkItemType"], state=work_item["fields"]["System.State"], url=work_item["url"]) for work_item in await self._work_items(responses)] return SourceMeasurement(value=value, entities=entities)
async def _entity(self, issue) -> Entity: """Create an entity from an issue.""" return Entity( key=issue["key"], url=await self.__issue_landing_url(issue["key"]), message=issue["message"], severity=issue.get("severity", "no severity").lower(), type=issue["type"].lower(), component=issue["component"])
def __card_to_entity(card, api_url: URL, board_slug: str, list_title: str) -> Entity: """Convert a card into a entity.""" return Entity( key=card["_id"], url=f"{api_url}/b/{card['boardId']}/{board_slug}/{card['_id']}", list=list_title, title=card["title"], due_date=card.get("dueAt", ""), date_last_activity=card["dateLastActivity"])
def _parse_entity( # pylint: disable=no-self-use self, dependency: Element, dependency_index: int, namespaces: Namespaces, landing_url: str) -> Entity: """Parse the entity from the dependency.""" file_path = dependency.findtext("ns:filePath", default="", namespaces=namespaces) sha1 = dependency.findtext("ns:sha1", namespaces=namespaces) # We can only generate an entity landing url if a sha1 is present in the XML, but unfortunately not all # dependencies have one, so check for it: entity_landing_url = f"{landing_url}#l{dependency_index + 1}_{sha1}" if sha1 else "" key = sha1 if sha1 else sha1_hash(file_path) return Entity(key=key, file_path=file_path, url=entity_landing_url)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: """Return a list of warnings.""" entities = [] for response in responses: entities.extend( [Entity( key=warning[self.KEY], package=warning[self.PACKAGE], installed=warning[self.INSTALLED], affected=warning[self.AFFECTED], vulnerability=warning[self.VULNERABILITY]) for warning in await response.json(content_type=None)]) return SourceMeasurement(entities=entities)
async def _entities(self, metrics: Dict[str, str]) -> List[Entity]: entities = [] api_values = self._data_model["sources"][self.source_type]["parameters"]["effort_types"]["api_values"] for effort_type in self.__effort_types(): effort_type_description = [param for param, api_key in api_values.items() if effort_type == api_key][0] entities.append( Entity( key=effort_type, effort_type=effort_type_description, effort=metrics[effort_type], url=await self.__effort_type_landing_url(effort_type))) return entities
def __entity(sprint: Sprint, sprint_points: SprintPoints, velocity_type: str, entity_url: URL) -> Entity: """Create a sprint entity.""" sprint_id = str(sprint["id"]) committed = sprint_points["estimated"]["text"] completed = sprint_points["completed"]["text"] difference = str(float(sprint_points["completed"]["value"] - sprint_points["estimated"]["value"])) measured = dict(completed=completed, estimated=committed, difference=difference)[velocity_type] return Entity( key=sprint["id"], name=sprint["name"], goal=sprint.get("goal") or "", points_completed=completed, points_committed=committed, points_measured=measured, points_difference=difference, url=str(entity_url) + sprint_id)
async def _parse_entities(self, responses: SourceResponses) -> Entities: """Override to parse the work items from the WIQL query response.""" return Entities( Entity( key=work_item["id"], project=work_item["fields"]["System.TeamProject"], title=work_item["fields"]["System.Title"], work_item_type=work_item["fields"]["System.WorkItemType"], state=work_item["fields"]["System.State"], url=work_item["url"], ) for work_item in await self._work_items(responses))
async def _entities(self, metrics: dict[str, str]) -> Entities: """Extend to return ncloc per language, if the users picked ncloc to measure.""" if self._value_key() == "ncloc": # Our user picked non-commented lines of code (ncloc), so we can show the ncloc per language, skipping # languages the user wants to ignore return Entities( Entity(key=language, language=self.LANGUAGES.get(language, language), ncloc=ncloc) for language, ncloc in self.__language_ncloc(metrics)) return await super()._entities(metrics)