async def __safely_parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Parse the data from the responses, without failing.
This method should not be overridden because it makes sure that the parsing of source data never causes the
collector to fail.
"""
if responses.connection_error:
measurement = SourceMeasurement(total=None)
else:
try:
measurement = await self._parse_source_responses(responses)
except Exception: # pylint: disable=broad-except
measurement = SourceMeasurement(
parse_error=stable_traceback(traceback.format_exc()))
return measurement
class SnykSecurityWarnings(JSONFileSourceCollector):
"""Snyk collector for security warnings."""
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Parse the direct dependencies with vulnerabilities from the responses."""
selected_severities = self._parameter("severities")
severities: Dict[str, Set[Severity]] = {}
nr_vulnerabilities: Dict[str, int] = {}
example_vulnerability = {}
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", [])
for vulnerability in vulnerabilities:
if (severity := vulnerability["severity"]) not in selected_severities:
continue
dependency = vulnerability["from"][1] if len(vulnerability["from"]) > 1 else vulnerability["from"][0]
severities.setdefault(dependency, set()).add(severity)
nr_vulnerabilities[dependency] = nr_vulnerabilities.get(dependency, 0) + 1
path = " ➜ ".join(str(dependency) for dependency in vulnerability["from"])
example_vulnerability[dependency] = (vulnerability["id"], path)
entities = []
for dependency in severities:
entities.append(
Entity(
key=dependency, dependency=dependency,
nr_vulnerabilities=nr_vulnerabilities[dependency],
example_vulnerability=example_vulnerability[dependency][0],
url=f"https://snyk.io/vuln/{example_vulnerability[dependency][0]}",
example_path=example_vulnerability[dependency][1],
highest_severity=self.__highest_severity(severities[dependency])))
return SourceMeasurement(entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
test_results = cast(List[str], self._parameter("test_result"))
test_run_names_to_include = cast(List[str], self._parameter("test_run_names_to_include")) or ["all"]
test_run_states_to_include = [
value.lower() for value in self._parameter("test_run_states_to_include")] or ["all"]
runs = (await responses[0].json()).get("value", [])
highest_build: Dict[str, TestRun] = defaultdict(TestRun)
for run in runs:
name = run.get("name", "Unknown test run name")
if test_run_names_to_include != ["all"] and \
not match_string_or_regular_expression(name, test_run_names_to_include):
continue
state = run.get("state", "Unknown test run state")
if test_run_states_to_include != ["all"] and state.lower() not in test_run_states_to_include:
continue
build_nr = int(run.get("build", {}).get("id", -1))
if build_nr < highest_build[name].build_nr:
continue
if build_nr > highest_build[name].build_nr:
highest_build[name] = TestRun(build_nr)
counted_tests = sum(run.get(test_result, 0) for test_result in test_results)
highest_build[name].test_count += counted_tests
highest_build[name].total_test_count += run.get("totalTests", 0)
highest_build[name].entities.append(
Entity(
key=run["id"], name=name, state=state, build_id=str(build_nr), url=run.get("webAccessUrl", ""),
started_date=run.get("startedDate", ""), completed_date=run.get("completedDate", ""),
counted_tests=str(counted_tests), incomplete_tests=str(run.get("incompleteTests", 0)),
not_applicable_tests=str(run.get("notApplicableTests", 0)),
passed_tests=str(run.get("passedTests", 0)), unanalyzed_tests=str(run.get("unanalyzedTests", 0)),
total_tests=str(run.get("totalTests", 0))))
test_count = sum(build.test_count for build in highest_build.values())
total_test_count = sum(build.total_test_count for build in highest_build.values())
test_runs = list(itertools.chain.from_iterable([build.entities for build in highest_build.values()]))
return SourceMeasurement(value=str(test_count), total=str(total_test_count), entities=test_runs)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
statuses = cast(List[str], self._parameter("test_result"))
json = await responses[0].json()
value = sum(int(json[status]) for status in statuses)
return SourceMeasurement(value=str(value),
total=str(json["overallTotal"]))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
entities = [
Entity(
key=branch["name"], name=branch["name"], commit_date=str(self._commit_datetime(branch).date()),
url=str(self._branch_landing_url(branch)))
for branch in await self._unmerged_branches(responses)]
return SourceMeasurement(entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the metrics."""
metrics = await self.__get_metrics(responses)
return SourceMeasurement(value=self._value(metrics),
total=self._total(metrics),
entities=await self._entities(metrics))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the sprint values from the responses."""
api_url = await self._api_url()
board_id = parse_qs(urlparse(str(
responses.api_url)).query)["rapidViewId"][0]
entity_url = URL(
f"{api_url}/secure/RapidBoard.jspa?rapidView={board_id}&view=reporting&chart=sprintRetrospective&sprint="
)
json = await responses[0].json()
# The JSON contains a list with sprints and a dict with the committed and completed points, indexed by sprint id
sprints, points = json["sprints"], json["velocityStatEntries"]
velocity_type = str(self._parameter("velocity_type"))
nr_sprints = int(str(self._parameter("velocity_sprints")))
# Get the points, either completed, committed, or completed minus committed, as determined by the velocity type
sprint_values = [
self.__velocity(points[str(sprint["id"])], velocity_type)
for sprint in sprints[:nr_sprints]
]
entities = Entities(
self.__entity(sprint, points[str(sprint["id"])], velocity_type,
entity_url) for sprint in sprints)
return SourceMeasurement(
value=str(round(sum(sprint_values) /
len(sprint_values))) if sprint_values else "0",
entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
entities: Dict[str, Entity] = {}
tag_re = re.compile(r"<[^>]*>")
risks = cast(List[str], self._parameter("risks"))
for alert in await self.__alerts(responses, risks):
ids = [
alert.findtext(id_tag, default="")
for id_tag in ("alert", "pluginid", "cweid", "wascid",
"sourceid")
]
name = alert.findtext("name", default="")
description = tag_re.sub("", alert.findtext("desc", default=""))
risk = alert.findtext("riskdesc", default="")
for alert_instance in alert.findall("./instances/instance"):
method = alert_instance.findtext("method", default="")
uri = self.__stable(
hashless(URL(alert_instance.findtext("uri", default=""))))
key = md5_hash(f"{':'.join(ids)}:{method}:{uri}")
entities[key] = Entity(
key=key,
old_key=md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}"),
name=name,
description=description,
uri=uri,
location=f"{method} {uri}",
risk=risk)
return SourceMeasurement(entities=list(entities.values()))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
entities = []
severities = self._parameter("severities")
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", []) if isinstance(
json, dict) else []
for vulnerability in vulnerabilities:
if vulnerability["severity"].lower() not in severities:
continue
package_include = " ➜ ".join([str(package) for package in vulnerability["from"][1:]]) \
if isinstance(vulnerability["from"], list) else vulnerability["from"]
fix = ", ".join([str(package) for package in vulnerability["fixedIn"]]) \
if isinstance(vulnerability["fixedIn"], list) else vulnerability["fixedIn"]
key = md5_hash(f'{vulnerability["id"]}:{package_include}')
entities.append(
Entity(key=key,
cve=vulnerability["title"],
package=vulnerability["packageName"],
severity=vulnerability["severity"],
version=vulnerability['version'],
package_include=package_include,
fix=fix,
url=f"https://snyk.io/vuln/{vulnerability['id']}"))
return SourceMeasurement(entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
count = 0
total = 0
entities = []
test_results = cast(List[str], self._parameter("test_result"))
all_test_results = self._data_model["sources"][
self.source_type]["parameters"]["test_result"]["values"]
for response in responses:
tree = await parse_source_response_xml(response)
stats = tree.findall("statistics/total/stat")[1]
for test_result in all_test_results:
total += int(stats.get(test_result, 0))
if test_result in test_results:
count += int(stats.get(test_result, 0))
for test in tree.findall(
f".//test/status[@status='{test_result.upper()}']/.."
):
entities.append(
Entity(key=test.get("id", ""),
name=test.get("name", ""),
test_result=test_result))
return SourceMeasurement(value=str(count),
total=str(total),
entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
entities = [
self.__entity(transaction)
for transaction in await self.__slow_transactions(responses)
]
return SourceMeasurement(entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
valid, covered = 0, 0
for response in responses:
tree = await parse_source_response_xml(response)
valid += int(tree.get(f"{self.coverage_type}-valid", 0))
covered += int(tree.get(f"{self.coverage_type}-covered", 0))
return SourceMeasurement(value=str(valid - covered), total=str(valid))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
impact_levels = self._parameter("impact")
entity_attributes = []
for response in responses:
json = await response.json(content_type=None)
url = json["url"]
for violation in json.get("violations", []):
for node in violation.get("nodes", []):
if node.get("impact") not in impact_levels:
continue
entity_attributes.append(
dict(description=violation.get("description"),
element=node.get("html"),
help=violation.get("helpUrl"),
impact=node.get("impact"),
page=url,
url=url,
violation_type=violation.get("id")))
entities = [
Entity(key=md5_hash(",".join(
str(value) for value in attributes.values())),
**attributes) for attributes in entity_attributes
]
return SourceMeasurement(entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
jobs = await self.__jobs(responses)
entities = [
Entity(
key=job["id"], name=job["name"], url=job["web_url"], build_status=job["status"], branch=job["ref"],
stage=job["stage"], build_date=str(parse(job["created_at"]).date()))
for job in jobs]
return SourceMeasurement(entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
value = 0
entities: List[Entity] = []
for response in responses:
json = await response.json()
value += int(json.get("total", 0))
entities.extend([await self._entity(issue) for issue in json.get("issues", [])])
return SourceMeasurement(value=str(value), entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
stats = await responses[0].json()
severities = self._parameter("severities")
return SourceMeasurement(value=str(
sum(
stats.get(f"{severity.lower()}Severity", 0)
for severity in severities)))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the slow transactions from the responses."""
entities = [
self.__entity(transaction)
for transaction in await self.__slow_transactions(responses)
]
return SourceMeasurement(entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
valid, covered = 0, 0
for response in responses:
tree = ElementTree.fromstring(await response.text())
valid += int(tree.get(f"{self.coverage_type}-valid"))
covered += int(tree.get(f"{self.coverage_type}-covered"))
return SourceMeasurement(value=str(valid - covered), total=str(valid))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
trend_breaks = []
for response in responses:
trend_breaks.append(
int((await self._soup(response)).find(
id="trendbreak_stability").string))
return SourceMeasurement(value=str(min(trend_breaks)))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
value = str(len((await responses[0].json())["workItems"]))
entities = [
Entity(
key=work_item["id"], project=work_item["fields"]["System.TeamProject"],
title=work_item["fields"]["System.Title"], work_item_type=work_item["fields"]["System.WorkItemType"],
state=work_item["fields"]["System.State"], url=work_item["url"])
for work_item in await self._work_items(responses)]
return SourceMeasurement(value=value, entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the trend break percentage from the responses and return the minimum percentage."""
trend_breaks = []
for response in responses:
trend_breaks.append(
int((await self._soup(response)).find(
id="trendbreak_stability").string))
return SourceMeasurement(value=str(min(trend_breaks)))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
landing_url = await self._landing_url(responses)
entities = []
for response in responses:
tree, namespaces = await parse_source_response_xml_with_namespace(response, self.allowed_root_tags)
entities.extend(
[self._parse_entity(dependency, index, namespaces, landing_url) for (index, dependency)
in enumerate(self._dependencies(tree, namespaces))])
return SourceMeasurement(entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
security_types = self._parameter(self.types_parameter)
vulnerabilities = await super()._parse_source_responses(SourceResponses(responses=[responses[0]])) \
if "vulnerability" in security_types else SourceMeasurement()
if "security_hotspot" in security_types:
json = await responses[-1].json()
nr_hotspots = int(json.get("paging", {}).get("total", 0))
hotspots = [
await self.__entity(hotspot)
for hotspot in json.get("hotspots", [])
]
else:
nr_hotspots = 0
hotspots = []
return SourceMeasurement(
value=str(int(vulnerabilities.value or 0) + nr_hotspots),
entities=vulnerabilities.entities + hotspots)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
durations = []
for response in responses:
hours, minutes, seconds = [
int(part) for part in (await self._soup(response)).find(
id="duration").string.split(":", 2)
]
durations.append(60 * hours + minutes + round(seconds / 60.))
return SourceMeasurement(value=str(sum(durations)))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
tests = await self.__nr_of_tests(responses)
value = str(
sum(tests[test_result]
for test_result in self._parameter("test_result")))
test_results = self._data_model["sources"][
self.source_type]["parameters"]["test_result"]["values"]
total = str(sum(tests[test_result] for test_result in test_results))
return SourceMeasurement(value=value, total=total)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to get the issues from the responses."""
url = URL(str(self._parameter("url")))
json = await responses[0].json()
entities = Entities(
self._create_entity(issue, url)
for issue in json.get("issues", []) if self._include_issue(issue))
return SourceMeasurement(value=self._compute_value(entities),
entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
severities = cast(List[str], self._parameter("severities"))
count = 0
entities = []
for response in responses:
tree, namespaces = await parse_source_response_xml_with_namespace(response)
entities.extend(self.__violations(tree, namespaces, severities))
for severity in severities:
count += int(tree.findtext(f"./ns:{severity}-count", default="0", namespaces=namespaces))
return SourceMeasurement(value=str(count), entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the coverage from the JaCoCo XML."""
missed, covered = 0, 0
for response in responses:
tree = await parse_source_response_xml(response)
counters = [c for c in tree.findall("counter") if c.get("type", "").lower() == self.coverage_type]
if counters:
missed += int(counters[0].get("missed", 0))
covered += int(counters[0].get("covered", 0))
return SourceMeasurement(value=str(missed), total=str(missed + covered))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Return a list of warnings."""
entities = []
for response in responses:
entities.extend(
[Entity(
key=warning[self.KEY], package=warning[self.PACKAGE], installed=warning[self.INSTALLED],
affected=warning[self.AFFECTED], vulnerability=warning[self.VULNERABILITY])
for warning in await response.json(content_type=None)])
return SourceMeasurement(entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
elements = (await responses[0].json())["results"]["elements"]
coverage = [
element for element in elements
if element["name"].lower() == self.coverage_type
][0]
total = int(coverage["denominator"])
return SourceMeasurement(value=str(total - int(coverage["numerator"])),
total=str(total))
