async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the issues."""
value = 0
entities = Entities()
for response in responses:
json = await response.json()
value += int(json.get("total", 0))
entities.extend([await self._entity(issue) for issue in json.get("issues", [])])
return SourceMeasurement(value=str(value), entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the cards."""
api_url = await self._api_url()
board_slug = self._board["slug"]
entities = Entities()
for lst in self._lists:
for card in self._cards.get(lst["_id"], []):
entities.append(
self.__card_to_entity(card, api_url, board_slug,
lst["title"]))
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the dependencies from the XML."""
landing_url = await self._landing_url(responses)
entities = Entities()
for response in responses:
tree, namespaces = await parse_source_response_xml_with_namespace(
response, self.allowed_root_tags)
entities.extend([
self._parse_entity(dependency, index, namespaces, landing_url)
for (index, dependency
) in enumerate(self._dependencies(tree, namespaces))
])
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the jobs/pipelines."""
entities = Entities()
for job in (await responses[0].json())["value"]:
if not self._include_job(job):
continue
name = self.__job_name(job)
url = job["_links"]["web"]["href"]
build_status = self._latest_build_result(job)
build_date_time = self._latest_build_date_time(job)
entities.append(
Entity(key=name, name=name, url=url, build_date=str(build_date_time.date()), build_status=build_status)
)
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings from the JSON."""
entities = Entities()
for response in responses:
entities.extend([
Entity(
key=warning[self.KEY],
package=warning[self.PACKAGE],
installed=warning[self.INSTALLED],
affected=warning[self.AFFECTED],
vulnerability=warning[self.VULNERABILITY],
) for warning in await response.json(content_type=None)
])
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the Anchore security warnings."""
severities = self._parameter("severities")
entities = Entities()
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", []) if isinstance(json, dict) else []
entities.extend(
[
self._create_entity(vulnerability, response.filename)
for vulnerability in vulnerabilities
if vulnerability["severity"] in severities
]
)
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the violations."""
entity_attributes = []
for response in responses:
json = await response.json(content_type=None)
url = json["url"]
for violation in json.get("violations", []):
for node in violation.get("nodes", []):
tags = violation.get("tags", [])
impact = node.get("impact")
if self.__include_violation(impact, tags):
entity_attributes.append(
dict(
description=violation.get("description"),
element=node.get("html"),
help=violation.get("helpUrl"),
impact=impact,
page=url,
url=url,
tags=", ".join(sorted(tags)),
violation_type=violation.get("id"),
))
return Entities(
Entity(key=self.__create_key(attributes), **attributes)
for attributes in entity_attributes)
class SnykSecurityWarnings(JSONFileSourceCollector):
"""Snyk collector for security warnings."""
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Parse the direct dependencies with vulnerabilities from the responses."""
selected_severities = self._parameter("severities")
severities: dict[str, set[Severity]] = {}
nr_vulnerabilities: dict[str, int] = {}
example_vulnerability = {}
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", [])
for vulnerability in vulnerabilities:
if (severity := vulnerability["severity"]) not in selected_severities:
continue
dependency = vulnerability["from"][1] if len(vulnerability["from"]) > 1 else vulnerability["from"][0]
severities.setdefault(dependency, set()).add(severity)
nr_vulnerabilities[dependency] = nr_vulnerabilities.get(dependency, 0) + 1
path = " ➜ ".join(str(dependency) for dependency in vulnerability["from"])
example_vulnerability[dependency] = (vulnerability["id"], path)
entities = Entities()
for dependency in severities:
entities.append(
Entity(
key=dependency,
dependency=dependency,
nr_vulnerabilities=nr_vulnerabilities[dependency],
example_vulnerability=example_vulnerability[dependency][0],
url=f"https://snyk.io/vuln/{example_vulnerability[dependency][0]}",
example_path=example_vulnerability[dependency][1],
highest_severity=self.__highest_severity(severities[dependency]),
)
)
return entities
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the sprint values from the responses."""
api_url = await self._api_url()
board_id = parse_qs(urlparse(str(
responses.api_url)).query)["rapidViewId"][0]
entity_url = URL(
f"{api_url}/secure/RapidBoard.jspa?rapidView={board_id}&view=reporting&chart=sprintRetrospective&sprint="
)
json = await responses[0].json()
# The JSON contains a list with sprints and a dict with the committed and completed points, indexed by sprint id
sprints, points = json["sprints"], json["velocityStatEntries"]
velocity_type = str(self._parameter("velocity_type"))
nr_sprints = int(str(self._parameter("velocity_sprints")))
# Get the points, either completed, committed, or completed minus committed, as determined by the velocity type
sprint_values = [
self.__velocity(points[str(sprint["id"])], velocity_type)
for sprint in sprints[:nr_sprints]
]
entities = Entities(
self.__entity(sprint, points[str(sprint["id"])], velocity_type,
entity_url) for sprint in sprints)
return SourceMeasurement(
value=str(round(sum(sprint_values) /
len(sprint_values))) if sprint_values else "0",
entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the violations from the OJAudit XML."""
severities = cast(list[str], self._parameter("severities"))
count = 0
entities = Entities()
for response in responses:
tree, namespaces = await parse_source_response_xml_with_namespace(
response)
entities.extend(self.__violations(tree, namespaces, severities))
for severity in severities:
count += int(
tree.findtext(f"./ns:{severity}-count",
default="0",
namespaces=namespaces))
return SourceMeasurement(value=str(count), entities=entities)
def __violations(self, tree: Element, namespaces: Namespaces,
severities: list[str]) -> Entities:
"""Return the violations."""
models = self.__model_file_paths(tree, namespaces)
violation_elements = tree.findall(".//ns:violation", namespaces)
violations = Entities()
for element in violation_elements:
violation = self.__violation(element, namespaces, models,
severities)
if violation is not None:
violations.append(violation)
# Add the duplication counts
for violation in violations:
violation["count"] = str(self.violation_counts[str(
violation["key"])])
return violations
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings from the OpenVAS XML."""
entities = Entities()
severities = cast(list[str], self._parameter("severities"))
for response in responses:
tree = await parse_source_response_xml(response)
entities.extend([
Entity(
key=result.attrib["id"],
name=result.findtext("name", default=""),
description=result.findtext("description", default=""),
host=result.findtext("host", default=""),
port=result.findtext("port", default=""),
severity=result.findtext("threat", default=""),
) for result in self.__results(tree, severities)
])
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the merge requests."""
merge_requests = []
for response in responses:
merge_requests.extend(await response.json())
return Entities(
self._create_entity(mr) for mr in merge_requests
if self._include_merge_request(mr))
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the issues from the JSON."""
json = await responses[0].json()
cards = json["cards"]
lists = {lst["id"]: lst["name"] for lst in json["lists"]}
return Entities(
self.__card_to_entity(card, lists) for card in cards
if not self.__ignore_card(card, lists))
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings from the JSON."""
entities = Entities()
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", [])
for vulnerability in vulnerabilities:
key = md5_hash(
f'{vulnerability["title"]}:{vulnerability["description"]}')
entities.append(
Entity(
key=key,
title=vulnerability["title"],
description=vulnerability["description"],
severity=vulnerability["severity"],
))
return entities
async def _entities(self, metrics: dict[str, str]) -> Entities:
"""Override to return the effort entities."""
entities = Entities()
api_values = self._data_model["sources"][
self.source_type]["parameters"]["effort_types"]["api_values"]
for effort_type in self.__effort_types():
effort_type_description = [
param for param, api_key in api_values.items()
if effort_type == api_key
][0]
entities.append(
Entity(
key=effort_type,
effort_type=effort_type_description,
effort=metrics[effort_type],
url=await self.__effort_type_landing_url(effort_type),
))
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the merge requests from the responses."""
merge_requests = []
for response in responses:
merge_requests.extend((await response.json())["value"])
landing_url = (await self._landing_url(responses)).rstrip("s")
return Entities(
self._create_entity(mr, landing_url) for mr in merge_requests
if self._include_merge_request(mr))
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to get the unmerged branches from the unmerged branches method that subclasses should implement."""
return Entities(
Entity(
key=branch["name"],
name=branch["name"],
commit_date=str(self._commit_datetime(branch).date()),
url=str(self._branch_landing_url(branch)),
) for branch in await self._unmerged_branches(responses))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the tests from the Robot Framework XML."""
count = 0
total = 0
entities = Entities()
test_results = cast(list[str], self._parameter("test_result"))
all_test_results = self._data_model["sources"][self.source_type]["parameters"]["test_result"]["values"]
for response in responses:
tree = await parse_source_response_xml(response)
stats = tree.findall("statistics/total/stat")[1]
for test_result in all_test_results:
total += int(stats.get(test_result, 0))
if test_result in test_results:
count += int(stats.get(test_result, 0))
for test in tree.findall(f".//test/status[@status='{test_result.upper()}']/.."):
entities.append(
Entity(key=test.get("id", ""), name=test.get("name", ""), test_result=test_result)
)
return SourceMeasurement(value=str(count), total=str(total), entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to get the issues from the responses."""
url = URL(str(self._parameter("url")))
json = await responses[0].json()
entities = Entities(
self._create_entity(issue, url)
for issue in json.get("issues", []) if self._include_issue(issue))
return SourceMeasurement(value=self._compute_value(entities),
entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the work items from the WIQL query response."""
return Entities(
Entity(
key=work_item["id"],
project=work_item["fields"]["System.TeamProject"],
title=work_item["fields"]["System.Title"],
work_item_type=work_item["fields"]["System.WorkItemType"],
state=work_item["fields"]["System.State"],
url=work_item["url"],
) for work_item in await self._work_items(responses))
async def _entities(self, metrics: dict[str, str]) -> Entities:
"""Extend to return ncloc per language, if the users picked ncloc to measure."""
if self._value_key() == "ncloc":
# Our user picked non-commented lines of code (ncloc), so we can show the ncloc per language, skipping
# languages the user wants to ignore
return Entities(
Entity(key=language,
language=self.LANGUAGES.get(language, language),
ncloc=ncloc)
for language, ncloc in self.__language_ncloc(metrics))
return await super()._entities(metrics)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the jobs."""
return Entities([
Entity(
key=job["name"],
name=job["name"],
url=job["url"],
build_status=self._build_status(job),
build_date=str(self._build_datetime(job).date())
if self._build_datetime(job) > datetime.min else "",
) for job in self.__jobs((await responses[0].json())["jobs"])
])
def __init__(self,
*,
value: Value = None,
total: Value = "100",
entities: Entities = None,
parse_error: ErrorMessage = None) -> None:
self.value = str(
len(entities)) if value is None and entities is not None else value
self.total = total
self.entities = entities[:self.MAX_ENTITIES] if entities else Entities(
)
self.parse_error = parse_error
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings."""
severities = self._parameter("severities")
confidence_levels = self._parameter("confidence_levels")
entities = Entities()
for response in responses:
entities.extend([
Entity(
key=
f'{warning["test_id"]}:{warning["filename"]}:{warning["line_number"]}',
location=f'{warning["filename"]}:{warning["line_number"]}',
issue_text=warning["issue_text"],
issue_severity=warning["issue_severity"].capitalize(),
issue_confidence=warning["issue_confidence"].capitalize(),
more_info=warning["more_info"],
) for warning in (await response.json(
content_type=None)).get("results", [])
if warning["issue_severity"].lower() in severities
and warning["issue_confidence"].lower() in confidence_levels
])
return entities
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the LOC from the JSON responses."""
loc = 0
entities = Entities()
languages_to_ignore = self._parameter("languages_to_ignore")
for response in responses:
for key, value in (await response.json(content_type=None)).items():
if key not in ("header", "SUM"
) and not match_string_or_regular_expression(
key, languages_to_ignore):
loc += value["code"]
entities.append(
Entity(
key=key,
language=key,
blank=str(value["blank"]),
comment=str(value["comment"]),
code=str(value["code"]),
nr_files=str(value["nFiles"]),
))
return SourceMeasurement(value=str(loc), entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the dependencies from the JSON."""
installed_dependencies: list[dict[str, str]] = []
for response in responses:
installed_dependencies.extend(await response.json(content_type=None))
return Entities(
Entity(
key=f'{dependency["name"]}@{dependency.get("version", "?")}',
name=dependency["name"],
version=dependency.get("version", "unknown"),
latest=dependency.get("latest_version", "unknown"),
)
for dependency in installed_dependencies
)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the dependencies from the JSOM."""
installed_dependencies: dict[str, dict[str, str]] = {}
for response in responses:
installed_dependencies.update(await
response.json(content_type=None))
return Entities(
Entity(
key=f'{dependency}@{versions.get("current", "?")}',
name=dependency,
current=versions.get("current", "unknown"),
wanted=versions.get("wanted", "unknown"),
latest=versions.get("latest", "unknown"),
) for dependency, versions in installed_dependencies.items())
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the tests from the JUnit XML."""
entities = Entities()
test_statuses_to_count = cast(list[str],
self._parameter("test_result"))
junit_status_nodes = dict(errored="error",
failed="failure",
skipped="skipped")
total = 0
for response in responses:
tree = await parse_source_response_xml(response)
for test_case in tree.findall(".//testcase"):
for test_result, junit_status_node in junit_status_nodes.items(
):
if test_case.find(junit_status_node) is not None:
break
else:
test_result = "passed"
if test_result in test_statuses_to_count:
entities.append(self.__entity(test_case, test_result))
total += 1
return SourceMeasurement(entities=entities, total=str(total))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Get the metric entities from the responses."""
status_to_count = self._parameter("status")
landing_url = await self._landing_url(responses)
metrics_and_entities = await self.__get_metrics_and_entities(
responses[0])
entities = Entities()
for metric, entity in metrics_and_entities:
recent_measurements: Measurements = cast(
Measurements, metric.get("recent_measurements", []))
status, value = self.__get_status_and_value(
metric, recent_measurements[-1] if recent_measurements else {})
if status in status_to_count:
entity[
"report_url"] = report_url = f"{landing_url}/{metric['report_uuid']}"
entity[
"subject_url"] = f"{report_url}#{metric['subject_uuid']}"
entity["metric_url"] = f"{report_url}#{entity['key']}"
entity["metric"] = str(
metric.get("name")
or self._data_model["metrics"][metric["type"]]["name"])
entity["status"] = status
unit = metric.get("unit") or self._data_model["metrics"][
metric["type"]]["unit"]
entity["measurement"] = f"{value or '?'} {unit}"
direction = str(
metric.get("direction") or
self._data_model["metrics"][metric["type"]]["direction"])
direction = {"<": "≦", ">": "≧"}.get(direction, direction)
target = metric.get("target") or self._data_model["metrics"][
metric["type"]]["target"]
entity["target"] = f"{direction} {target} {unit}"
entities.append(entity)
return SourceMeasurement(total=str(len(metrics_and_entities)),
entities=entities)