async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the violations."""
entity_attributes = []
for response in responses:
json = await response.json(content_type=None)
url = json["url"]
for violation in json.get("violations", []):
for node in violation.get("nodes", []):
tags = violation.get("tags", [])
impact = node.get("impact")
if self.__include_violation(impact, tags):
entity_attributes.append(
dict(
description=violation.get("description"),
element=node.get("html"),
help=violation.get("helpUrl"),
impact=impact,
page=url,
url=url,
tags=", ".join(sorted(tags)),
violation_type=violation.get("id"),
))
return Entities(
Entity(key=self.__create_key(attributes), **attributes)
for attributes in entity_attributes)
class SnykSecurityWarnings(JSONFileSourceCollector):
"""Snyk collector for security warnings."""
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Parse the direct dependencies with vulnerabilities from the responses."""
selected_severities = self._parameter("severities")
severities: dict[str, set[Severity]] = {}
nr_vulnerabilities: dict[str, int] = {}
example_vulnerability = {}
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", [])
for vulnerability in vulnerabilities:
if (severity := vulnerability["severity"]) not in selected_severities:
continue
dependency = vulnerability["from"][1] if len(vulnerability["from"]) > 1 else vulnerability["from"][0]
severities.setdefault(dependency, set()).add(severity)
nr_vulnerabilities[dependency] = nr_vulnerabilities.get(dependency, 0) + 1
path = " ➜ ".join(str(dependency) for dependency in vulnerability["from"])
example_vulnerability[dependency] = (vulnerability["id"], path)
entities = Entities()
for dependency in severities:
entities.append(
Entity(
key=dependency,
dependency=dependency,
nr_vulnerabilities=nr_vulnerabilities[dependency],
example_vulnerability=example_vulnerability[dependency][0],
url=f"https://snyk.io/vuln/{example_vulnerability[dependency][0]}",
example_path=example_vulnerability[dependency][1],
highest_severity=self.__highest_severity(severities[dependency]),
)
)
return entities
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the sprint values from the responses."""
api_url = await self._api_url()
board_id = parse_qs(urlparse(str(
responses.api_url)).query)["rapidViewId"][0]
entity_url = URL(
f"{api_url}/secure/RapidBoard.jspa?rapidView={board_id}&view=reporting&chart=sprintRetrospective&sprint="
)
json = await responses[0].json()
# The JSON contains a list with sprints and a dict with the committed and completed points, indexed by sprint id
sprints, points = json["sprints"], json["velocityStatEntries"]
velocity_type = str(self._parameter("velocity_type"))
nr_sprints = int(str(self._parameter("velocity_sprints")))
# Get the points, either completed, committed, or completed minus committed, as determined by the velocity type
sprint_values = [
self.__velocity(points[str(sprint["id"])], velocity_type)
for sprint in sprints[:nr_sprints]
]
entities = Entities(
self.__entity(sprint, points[str(sprint["id"])], velocity_type,
entity_url) for sprint in sprints)
return SourceMeasurement(
value=str(round(sum(sprint_values) /
len(sprint_values))) if sprint_values else "0",
entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the issues from the JSON."""
json = await responses[0].json()
cards = json["cards"]
lists = {lst["id"]: lst["name"] for lst in json["lists"]}
return Entities(
self.__card_to_entity(card, lists) for card in cards
if not self.__ignore_card(card, lists))
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the merge requests."""
merge_requests = []
for response in responses:
merge_requests.extend(await response.json())
return Entities(
self._create_entity(mr) for mr in merge_requests
if self._include_merge_request(mr))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the issues."""
value = 0
entities = Entities()
for response in responses:
json = await response.json()
value += int(json.get("total", 0))
entities.extend([await self._entity(issue) for issue in json.get("issues", [])])
return SourceMeasurement(value=str(value), entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the merge requests from the responses."""
merge_requests = []
for response in responses:
merge_requests.extend((await response.json())["value"])
landing_url = (await self._landing_url(responses)).rstrip("s")
return Entities(
self._create_entity(mr, landing_url) for mr in merge_requests
if self._include_merge_request(mr))
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to get the unmerged branches from the unmerged branches method that subclasses should implement."""
return Entities(
Entity(
key=branch["name"],
name=branch["name"],
commit_date=str(self._commit_datetime(branch).date()),
url=str(self._branch_landing_url(branch)),
) for branch in await self._unmerged_branches(responses))
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to get the issues from the responses."""
url = URL(str(self._parameter("url")))
json = await responses[0].json()
entities = Entities(
self._create_entity(issue, url)
for issue in json.get("issues", []) if self._include_issue(issue))
return SourceMeasurement(value=self._compute_value(entities),
entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the work items from the WIQL query response."""
return Entities(
Entity(
key=work_item["id"],
project=work_item["fields"]["System.TeamProject"],
title=work_item["fields"]["System.Title"],
work_item_type=work_item["fields"]["System.WorkItemType"],
state=work_item["fields"]["System.State"],
url=work_item["url"],
) for work_item in await self._work_items(responses))
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the cards."""
api_url = await self._api_url()
board_slug = self._board["slug"]
entities = Entities()
for lst in self._lists:
for card in self._cards.get(lst["_id"], []):
entities.append(
self.__card_to_entity(card, api_url, board_slug,
lst["title"]))
return entities
async def _entities(self, metrics: dict[str, str]) -> Entities:
"""Extend to return ncloc per language, if the users picked ncloc to measure."""
if self._value_key() == "ncloc":
# Our user picked non-commented lines of code (ncloc), so we can show the ncloc per language, skipping
# languages the user wants to ignore
return Entities(
Entity(key=language,
language=self.LANGUAGES.get(language, language),
ncloc=ncloc)
for language, ncloc in self.__language_ncloc(metrics))
return await super()._entities(metrics)
def __init__(self,
*,
value: Value = None,
total: Value = "100",
entities: Entities = None,
parse_error: ErrorMessage = None) -> None:
self.value = str(
len(entities)) if value is None and entities is not None else value
self.total = total
self.entities = entities[:self.MAX_ENTITIES] if entities else Entities(
)
self.parse_error = parse_error
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the jobs."""
return Entities([
Entity(
key=job["name"],
name=job["name"],
url=job["url"],
build_status=self._build_status(job),
build_date=str(self._build_datetime(job).date())
if self._build_datetime(job) > datetime.min else "",
) for job in self.__jobs((await responses[0].json())["jobs"])
])
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the dependencies from the XML."""
landing_url = await self._landing_url(responses)
entities = Entities()
for response in responses:
tree, namespaces = await parse_source_response_xml_with_namespace(
response, self.allowed_root_tags)
entities.extend([
self._parse_entity(dependency, index, namespaces, landing_url)
for (index, dependency
) in enumerate(self._dependencies(tree, namespaces))
])
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings from the JSON."""
entities = Entities()
for response in responses:
entities.extend([
Entity(
key=warning[self.KEY],
package=warning[self.PACKAGE],
installed=warning[self.INSTALLED],
affected=warning[self.AFFECTED],
vulnerability=warning[self.VULNERABILITY],
) for warning in await response.json(content_type=None)
])
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the dependencies from the JSOM."""
installed_dependencies: dict[str, dict[str, str]] = {}
for response in responses:
installed_dependencies.update(await
response.json(content_type=None))
return Entities(
Entity(
key=f'{dependency}@{versions.get("current", "?")}',
name=dependency,
current=versions.get("current", "unknown"),
wanted=versions.get("wanted", "unknown"),
latest=versions.get("latest", "unknown"),
) for dependency, versions in installed_dependencies.items())
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the jobs/pipelines."""
entities = Entities()
for job in (await responses[0].json())["value"]:
if not self._include_job(job):
continue
name = self.__job_name(job)
url = job["_links"]["web"]["href"]
build_status = self._latest_build_result(job)
build_date_time = self._latest_build_date_time(job)
entities.append(
Entity(key=name, name=name, url=url, build_date=str(build_date_time.date()), build_status=build_status)
)
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the dependencies from the JSON."""
installed_dependencies: list[dict[str, str]] = []
for response in responses:
installed_dependencies.extend(await response.json(content_type=None))
return Entities(
Entity(
key=f'{dependency["name"]}@{dependency.get("version", "?")}',
name=dependency["name"],
version=dependency.get("version", "unknown"),
latest=dependency.get("latest_version", "unknown"),
)
for dependency in installed_dependencies
)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the Anchore security warnings."""
severities = self._parameter("severities")
entities = Entities()
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", []) if isinstance(json, dict) else []
entities.extend(
[
self._create_entity(vulnerability, response.filename)
for vulnerability in vulnerabilities
if vulnerability["severity"] in severities
]
)
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the jobs from the responses."""
return Entities(
[
Entity(
key=job["id"],
name=job["name"],
url=job["web_url"],
build_status=job["status"],
branch=job["ref"],
stage=job["stage"],
build_date=str(parse(job["created_at"]).date()),
)
for job in await self.__jobs(responses)
]
)
def __violations(self, tree: Element, namespaces: Namespaces,
severities: list[str]) -> Entities:
"""Return the violations."""
models = self.__model_file_paths(tree, namespaces)
violation_elements = tree.findall(".//ns:violation", namespaces)
violations = Entities()
for element in violation_elements:
violation = self.__violation(element, namespaces, models,
severities)
if violation is not None:
violations.append(violation)
# Add the duplication counts
for violation in violations:
violation["count"] = str(self.violation_counts[str(
violation["key"])])
return violations
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the violations from the OJAudit XML."""
severities = cast(list[str], self._parameter("severities"))
count = 0
entities = Entities()
for response in responses:
tree, namespaces = await parse_source_response_xml_with_namespace(
response)
entities.extend(self.__violations(tree, namespaces, severities))
for severity in severities:
count += int(
tree.findtext(f"./ns:{severity}-count",
default="0",
namespaces=namespaces))
return SourceMeasurement(value=str(count), entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the test runs."""
test_results = cast(list[str], self._parameter("test_result"))
test_run_names_to_include = cast(list[str], self._parameter("test_run_names_to_include")) or ["all"]
test_run_states_to_include = [value.lower() for value in self._parameter("test_run_states_to_include")] or [
"all"
]
runs = (await responses[0].json()).get("value", [])
highest_build: dict[str, TestRun] = defaultdict(TestRun)
for run in runs:
name = run.get("name", "Unknown test run name")
if test_run_names_to_include != ["all"] and not match_string_or_regular_expression(
name, test_run_names_to_include
):
continue
state = run.get("state", "Unknown test run state")
if test_run_states_to_include != ["all"] and state.lower() not in test_run_states_to_include:
continue
build_nr = int(run.get("build", {}).get("id", -1))
if build_nr < highest_build[name].build_nr:
continue
if build_nr > highest_build[name].build_nr:
highest_build[name] = TestRun(build_nr)
counted_tests = sum(run.get(test_result, 0) for test_result in test_results)
highest_build[name].test_count += counted_tests
highest_build[name].total_test_count += run.get("totalTests", 0)
highest_build[name].entities.append(
Entity(
key=run["id"],
name=name,
state=state,
build_id=str(build_nr),
url=run.get("webAccessUrl", ""),
started_date=run.get("startedDate", ""),
completed_date=run.get("completedDate", ""),
counted_tests=str(counted_tests),
incomplete_tests=str(run.get("incompleteTests", 0)),
not_applicable_tests=str(run.get("notApplicableTests", 0)),
passed_tests=str(run.get("passedTests", 0)),
unanalyzed_tests=str(run.get("unanalyzedTests", 0)),
total_tests=str(run.get("totalTests", 0)),
)
)
test_count = sum(build.test_count for build in highest_build.values())
total_test_count = sum(build.total_test_count for build in highest_build.values())
test_runs = Entities(itertools.chain.from_iterable([build.entities for build in highest_build.values()]))
return SourceMeasurement(value=str(test_count), total=str(total_test_count), entities=test_runs)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the CSV and create the entities."""
entity_attributes = [
dict(
url=str(row["URL"]),
violation_type=row["Violation Type"],
impact=row["Impact"],
element=row["DOM Element"],
page=re.sub(r"http[s]?://[^/]+", "", row["URL"]),
description=row["Messages"],
help=row["Help"],
) for row in await self.__parse_csv(responses)
]
return Entities(
Entity(key=md5_hash(",".join(
str(value) for value in attributes.values())),
**attributes) for attributes in entity_attributes)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings from the JSON."""
entities = Entities()
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", [])
for vulnerability in vulnerabilities:
key = md5_hash(
f'{vulnerability["title"]}:{vulnerability["description"]}')
entities.append(
Entity(
key=key,
title=vulnerability["title"],
description=vulnerability["description"],
severity=vulnerability["severity"],
))
return entities
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings from the OpenVAS XML."""
entities = Entities()
severities = cast(list[str], self._parameter("severities"))
for response in responses:
tree = await parse_source_response_xml(response)
entities.extend([
Entity(
key=result.attrib["id"],
name=result.findtext("name", default=""),
description=result.findtext("description", default=""),
host=result.findtext("host", default=""),
port=result.findtext("port", default=""),
severity=result.findtext("threat", default=""),
) for result in self.__results(tree, severities)
])
return entities
async def _entities(self, metrics: dict[str, str]) -> Entities:
"""Override to return the effort entities."""
entities = Entities()
api_values = self._data_model["sources"][
self.source_type]["parameters"]["effort_types"]["api_values"]
for effort_type in self.__effort_types():
effort_type_description = [
param for param, api_key in api_values.items()
if effort_type == api_key
][0]
entities.append(
Entity(
key=effort_type,
effort_type=effort_type_description,
effort=metrics[effort_type],
url=await self.__effort_type_landing_url(effort_type),
))
return entities
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to parse the tests from the Robot Framework XML."""
count = 0
total = 0
entities = Entities()
test_results = cast(list[str], self._parameter("test_result"))
all_test_results = self._data_model["sources"][self.source_type]["parameters"]["test_result"]["values"]
for response in responses:
tree = await parse_source_response_xml(response)
stats = tree.findall("statistics/total/stat")[1]
for test_result in all_test_results:
total += int(stats.get(test_result, 0))
if test_result in test_results:
count += int(stats.get(test_result, 0))
for test in tree.findall(f".//test/status[@status='{test_result.upper()}']/.."):
entities.append(
Entity(key=test.get("id", ""), name=test.get("name", ""), test_result=test_result)
)
return SourceMeasurement(value=str(count), total=str(total), entities=entities)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the security warnings."""
severities = self._parameter("severities")
confidence_levels = self._parameter("confidence_levels")
entities = Entities()
for response in responses:
entities.extend([
Entity(
key=
f'{warning["test_id"]}:{warning["filename"]}:{warning["line_number"]}',
location=f'{warning["filename"]}:{warning["line_number"]}',
issue_text=warning["issue_text"],
issue_severity=warning["issue_severity"].capitalize(),
issue_confidence=warning["issue_confidence"].capitalize(),
more_info=warning["more_info"],
) for warning in (await response.json(
content_type=None)).get("results", [])
if warning["issue_severity"].lower() in severities
and warning["issue_confidence"].lower() in confidence_levels
])
return entities