async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
count = 0
total = 0
entities = []
test_results = cast(List[str], self._parameter("test_result"))
all_test_results = self._data_model["sources"][
self.source_type]["parameters"]["test_result"]["values"]
for response in responses:
tree = await parse_source_response_xml(response)
stats = tree.findall("statistics/total/stat")[1]
for test_result in all_test_results:
total += int(stats.get(test_result, 0))
if test_result in test_results:
count += int(stats.get(test_result, 0))
for test in tree.findall(
f".//test/status[@status='{test_result.upper()}']/.."
):
entities.append(
Entity(key=test.get("id", ""),
name=test.get("name", ""),
test_result=test_result))
return SourceMeasurement(value=str(count),
total=str(total),
entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
entities = []
severities = self._parameter("severities")
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", []) if isinstance(
json, dict) else []
for vulnerability in vulnerabilities:
if vulnerability["severity"].lower() not in severities:
continue
package_include = " ➜ ".join([str(package) for package in vulnerability["from"][1:]]) \
if isinstance(vulnerability["from"], list) else vulnerability["from"]
fix = ", ".join([str(package) for package in vulnerability["fixedIn"]]) \
if isinstance(vulnerability["fixedIn"], list) else vulnerability["fixedIn"]
key = md5_hash(f'{vulnerability["id"]}:{package_include}')
entities.append(
Entity(key=key,
cve=vulnerability["title"],
package=vulnerability["packageName"],
severity=vulnerability["severity"],
version=vulnerability['version'],
package_include=package_include,
fix=fix,
url=f"https://snyk.io/vuln/{vulnerability['id']}"))
return SourceMeasurement(entities=entities)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
entities: Dict[str, Entity] = {}
tag_re = re.compile(r"<[^>]*>")
risks = cast(List[str], self._parameter("risks"))
for alert in await self.__alerts(responses, risks):
ids = [
alert.findtext(id_tag, default="")
for id_tag in ("alert", "pluginid", "cweid", "wascid",
"sourceid")
]
name = alert.findtext("name", default="")
description = tag_re.sub("", alert.findtext("desc", default=""))
risk = alert.findtext("riskdesc", default="")
for alert_instance in alert.findall("./instances/instance"):
method = alert_instance.findtext("method", default="")
uri = self.__stable(
hashless(URL(alert_instance.findtext("uri", default=""))))
key = md5_hash(f"{':'.join(ids)}:{method}:{uri}")
entities[key] = Entity(
key=key,
old_key=md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}"),
name=name,
description=description,
uri=uri,
location=f"{method} {uri}",
risk=risk)
return SourceMeasurement(entities=list(entities.values()))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
entities = [
Entity(
key=job["name"], name=job["name"], url=job["url"], build_status=self._build_status(job),
build_date=str(self._build_datetime(job).date()) if self._build_datetime(job) > datetime.min else "")
for job in self.__jobs((await responses[0].json())["jobs"])]
return SourceMeasurement(entities=entities)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
test_results = cast(List[str], self._parameter("test_result"))
test_run_names_to_include = cast(List[str], self._parameter("test_run_names_to_include")) or ["all"]
test_run_states_to_include = [
value.lower() for value in self._parameter("test_run_states_to_include")] or ["all"]
runs = (await responses[0].json()).get("value", [])
highest_build: Dict[str, TestRun] = defaultdict(TestRun)
for run in runs:
name = run.get("name", "Unknown test run name")
if test_run_names_to_include != ["all"] and \
not match_string_or_regular_expression(name, test_run_names_to_include):
continue
state = run.get("state", "Unknown test run state")
if test_run_states_to_include != ["all"] and state.lower() not in test_run_states_to_include:
continue
build_nr = int(run.get("build", {}).get("id", -1))
if build_nr < highest_build[name].build_nr:
continue
if build_nr > highest_build[name].build_nr:
highest_build[name] = TestRun(build_nr)
counted_tests = sum(run.get(test_result, 0) for test_result in test_results)
highest_build[name].test_count += counted_tests
highest_build[name].total_test_count += run.get("totalTests", 0)
highest_build[name].entities.append(
Entity(
key=run["id"], name=name, state=state, build_id=str(build_nr), url=run.get("webAccessUrl", ""),
started_date=run.get("startedDate", ""), completed_date=run.get("completedDate", ""),
counted_tests=str(counted_tests), incomplete_tests=str(run.get("incompleteTests", 0)),
not_applicable_tests=str(run.get("notApplicableTests", 0)),
passed_tests=str(run.get("passedTests", 0)), unanalyzed_tests=str(run.get("unanalyzedTests", 0)),
total_tests=str(run.get("totalTests", 0))))
test_count = sum(build.test_count for build in highest_build.values())
total_test_count = sum(build.total_test_count for build in highest_build.values())
test_runs = list(itertools.chain.from_iterable([build.entities for build in highest_build.values()]))
return SourceMeasurement(value=str(test_count), total=str(total_test_count), entities=test_runs)
def __entity(case_node, case_result: str) -> Entity:
"""Transform a test case into a test case entity."""
name = case_node.get("name", "<nameless test case>")
return Entity(key=name,
name=name,
class_name=case_node.get("classname", ""),
test_result=case_result)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
entities = [
Entity(
key=branch["name"], name=branch["name"], commit_date=str(self._commit_datetime(branch).date()),
url=str(self._branch_landing_url(branch)))
for branch in await self._unmerged_branches(responses)]
return SourceMeasurement(entities=entities)
def setUp(self):
"""Prepare the security warnings metric and sources."""
super().setUp()
self.sources = dict(
source_id=dict(type="snyk", parameters=dict(url="snyk.json")))
self.metric = dict(type="security_warnings",
sources=self.sources,
addition="sum")
self.direct_dependency = "[email protected]"
self.direct_dependency_key = Entity.safe_entity_key(
self.direct_dependency)
self.direct_dependency_path = [
"package.json@*", self.direct_dependency
]
self.vulnerabilities_json = dict(vulnerabilities=[{
"id":
"SNYK-JS-ACORN-559469",
"severity":
"low",
"from":
self.direct_dependency_path + ["[email protected]", "[email protected]"],
}])
self.expected_entity = dict(
key=self.direct_dependency_key,
dependency=self.direct_dependency,
nr_vulnerabilities=1,
example_vulnerability="SNYK-JS-ACORN-559469",
url="https://snyk.io/vuln/SNYK-JS-ACORN-559469",
example_path=
"package.json@* ➜ [email protected] ➜ [email protected] ➜ [email protected]",
highest_severity="low",
)
def entity( # pylint: disable=too-many-arguments
self,
component: str,
entity_type: str,
severity: str = None,
resolution: str = None,
vulnerability_probability: str = None,
creation_date: str = None,
update_date: str = None) -> Entity:
"""Create an entity."""
url = self.hotspot_landing_url.format(component) if entity_type == "security_hotspot" else \
self.issue_landing_url.format(component)
entity = Entity(key=component,
component=component,
message=component,
type=entity_type,
url=url)
if severity is not None:
entity["severity"] = severity
if resolution is not None:
entity["resolution"] = resolution
if vulnerability_probability is not None:
entity["vulnerability_probability"] = vulnerability_probability
entity["creation_date"] = creation_date
entity["update_date"] = update_date
return entity
class SnykSecurityWarnings(JSONFileSourceCollector):
"""Snyk collector for security warnings."""
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Parse the direct dependencies with vulnerabilities from the responses."""
selected_severities = self._parameter("severities")
severities: Dict[str, Set[Severity]] = {}
nr_vulnerabilities: Dict[str, int] = {}
example_vulnerability = {}
for response in responses:
json = await response.json(content_type=None)
vulnerabilities = json.get("vulnerabilities", [])
for vulnerability in vulnerabilities:
if (severity := vulnerability["severity"]) not in selected_severities:
continue
dependency = vulnerability["from"][1] if len(vulnerability["from"]) > 1 else vulnerability["from"][0]
severities.setdefault(dependency, set()).add(severity)
nr_vulnerabilities[dependency] = nr_vulnerabilities.get(dependency, 0) + 1
path = " ➜ ".join(str(dependency) for dependency in vulnerability["from"])
example_vulnerability[dependency] = (vulnerability["id"], path)
entities = []
for dependency in severities:
entities.append(
Entity(
key=dependency, dependency=dependency,
nr_vulnerabilities=nr_vulnerabilities[dependency],
example_vulnerability=example_vulnerability[dependency][0],
url=f"https://snyk.io/vuln/{example_vulnerability[dependency][0]}",
example_path=example_vulnerability[dependency][1],
highest_severity=self.__highest_severity(severities[dependency])))
return SourceMeasurement(entities=entities)
def entity( # pylint: disable=too-many-arguments
component: str,
entity_type: str,
severity: str = None,
resolution: str = None,
review_priority: str = None,
creation_date: str = None,
update_date: str = None,
) -> Entity:
"""Create an entity."""
url = (
f"https://sonarqube/security_hotspots?id=id&hotspots={component}&branch=master"
if entity_type == "security_hotspot"
else f"https://sonarqube/project/issues?id=id&issues={component}&open={component}&branch=master"
)
entity = Entity(
key=component,
component=component,
message=component,
type=entity_type,
url=url,
creation_date=creation_date,
update_date=update_date,
)
if severity is not None:
entity["severity"] = severity
if resolution is not None:
entity["resolution"] = resolution
if review_priority is not None:
entity["review_priority"] = review_priority
return entity
def __violation(self, violation: Element, namespaces: Namespaces,
models: ModelFilePaths,
severities: list[str]) -> Optional[Entity]:
"""Return the violation as entity."""
location = violation.find("./ns:location", namespaces)
if not location:
raise SourceCollectorException(
f"OJAudit violation {violation} has no location element")
severity = violation.findtext("./ns:values/ns:value",
default="",
namespaces=namespaces)
if severities and severity not in severities:
return None
message = violation.findtext("ns:message",
default="",
namespaces=namespaces)
line_number = violation.findtext(".//ns:line-number",
namespaces=namespaces)
column_offset = violation.findtext(".//ns:column-offset",
namespaces=namespaces)
model = models[location.get("model", "")]
component = f"{model}:{line_number}:{column_offset}"
key = sha1_hash(f"{message}:{component}")
entity = Entity(key=key,
severity=severity,
message=message,
component=component)
if entity["key"] in self.violation_counts:
self.violation_counts[entity["key"]] += 1
return None # Ignore duplicate violation
self.violation_counts[entity["key"]] = 1
return entity
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the violations."""
entity_attributes = []
for response in responses:
json = await response.json(content_type=None)
url = json["url"]
for violation in json.get("violations", []):
for node in violation.get("nodes", []):
tags = violation.get("tags", [])
impact = node.get("impact")
if self.__include_violation(impact, tags):
entity_attributes.append(
dict(
description=violation.get("description"),
element=node.get("html"),
help=violation.get("helpUrl"),
impact=impact,
page=url,
url=url,
tags=", ".join(sorted(tags)),
violation_type=violation.get("id"),
))
return Entities(
Entity(key=self.__create_key(attributes), **attributes)
for attributes in entity_attributes)
async def _parse_source_responses(
self, responses: SourceResponses) -> SourceMeasurement:
impact_levels = self._parameter("impact")
entity_attributes = []
for response in responses:
json = await response.json(content_type=None)
url = json["url"]
for violation in json.get("violations", []):
for node in violation.get("nodes", []):
if node.get("impact") not in impact_levels:
continue
entity_attributes.append(
dict(description=violation.get("description"),
element=node.get("html"),
help=violation.get("helpUrl"),
impact=node.get("impact"),
page=url,
url=url,
violation_type=violation.get("id")))
entities = [
Entity(key=md5_hash(",".join(
str(value) for value in attributes.values())),
**attributes) for attributes in entity_attributes
]
return SourceMeasurement(entities=entities)
class JiraIssues(SourceCollector):
"""Jira collector for issues."""
SPRINT_NAME_RE = re.compile(r",name=(.*),startDate=")
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._field_ids = {}
async def _api_url(self) -> URL:
"""Extend to get the fields from Jira and create a field name to field id mapping."""
url = await super()._api_url()
fields_url = URL(f"{url}/rest/api/2/field")
response = (await super()._get_source_responses(fields_url))[0]
self._field_ids = {field["name"].lower(): field["id"] for field in await response.json()}
jql = str(self._parameter("jql", quote=True))
fields = self._fields()
return URL(f"{url}/rest/api/2/search?jql={jql}&fields={fields}&maxResults=500")
async def _landing_url(self, responses: SourceResponses) -> URL:
"""Extend to add the JQL query to the landing URL."""
url = await super()._landing_url(responses)
jql = str(self._parameter("jql", quote=True))
return URL(f"{url}/issues/?jql={jql}")
def _parameter(self, parameter_key: str, quote: bool = False) -> Union[str, List[str]]:
"""Extend to replace field names with field ids, if the parameter is a field."""
parameter_value = super()._parameter(parameter_key, quote)
if parameter_key.endswith("field"):
parameter_value = self._field_ids.get(str(parameter_value).lower(), parameter_value)
return parameter_value
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Override to get the issues from the responses."""
url = URL(str(self._parameter("url")))
json = await responses[0].json()
entities = [self._create_entity(issue, url) for issue in json.get("issues", []) if self._include_issue(issue)]
return SourceMeasurement(value=self._compute_value(entities), entities=entities)
@classmethod
def _compute_value(cls, entities: List[Entity]) -> Value: # pylint: disable=unused-argument
"""Allow subclasses to compute the value from the entities."""
return None
def _create_entity(self, issue: Dict, url: URL) -> Entity: # pylint: disable=no-self-use
"""Create an entity from a Jira issue."""
fields = issue["fields"]
entity_attributes = dict(
created=fields["created"],
priority=fields.get("priority", {}).get("name"),
status=fields.get("status", {}).get("name"),
summary=fields["summary"],
type=fields.get("issuetype", {}).get("name", "Unknown issue type"),
updated=fields.get("updated"),
url=f"{url}/browse/{issue['key']}",
)
if sprint_field_id := self._field_ids.get("sprint"):
entity_attributes["sprint"] = self.__get_sprint_names(fields.get(sprint_field_id) or [])
return Entity(key=issue["id"], **entity_attributes)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
jobs = await self.__jobs(responses)
entities = [
Entity(
key=job["id"], name=job["name"], url=job["web_url"], build_status=job["status"], branch=job["ref"],
stage=job["stage"], build_date=str(parse(job["created_at"]).date()))
for job in jobs]
return SourceMeasurement(entities=entities)
async def _entities(self, metrics: Dict[str, str]) -> List[Entity]:
if self._value_key() == "ncloc":
# Our user picked non-commented lines of code (ncloc), so we can show the ncloc per language, skipping
# languages the user wants to ignore
return [
Entity(key=language, language=self.LANGUAGES.get(language, language), ncloc=ncloc)
for language, ncloc in self.__language_ncloc(metrics)]
return await super()._entities(metrics)
async def __entity(self, hotspot) -> Entity:
return Entity(
key=hotspot["key"],
component=hotspot["component"],
message=hotspot["message"],
type="security_hotspot",
url=await self.__hotspot_landing_url(hotspot["key"]),
vulnerability_probability=hotspot["vulnerabilityProbability"].lower())
def __card_to_entity(card, lists) -> Entity:
"""Convert a card into a entity."""
return Entity(key=card["id"],
title=card["name"],
url=card["url"],
list=lists[card["idList"]],
due_date=card["due"],
date_last_activity=card["dateLastActivity"])
def __alert_instance_entity(self, ids, entity_kwargs, alert_instance) -> Entity:
"""Create an alert instance entity."""
method = alert_instance.findtext("method", default="")
uri = self.__stable_url(hashless(URL(alert_instance.findtext("uri", default=""))))
key = md5_hash(f"{':'.join(ids)}:{method}:{uri}")
old_key = md5_hash(f"{':'.join(ids[1:])}:{method}:{uri}")
location = f"{method} {uri}"
return Entity(key=key, old_key=old_key, uri=uri, location=location, **entity_kwargs)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to get the unmerged branches from the unmerged branches method that subclasses should implement."""
return Entities(
Entity(
key=branch["name"],
name=branch["name"],
commit_date=str(self._commit_datetime(branch).date()),
url=str(self._branch_landing_url(branch)),
) for branch in await self._unmerged_branches(responses))
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
value = str(len((await responses[0].json())["workItems"]))
entities = [
Entity(
key=work_item["id"], project=work_item["fields"]["System.TeamProject"],
title=work_item["fields"]["System.Title"], work_item_type=work_item["fields"]["System.WorkItemType"],
state=work_item["fields"]["System.State"], url=work_item["url"])
for work_item in await self._work_items(responses)]
return SourceMeasurement(value=value, entities=entities)
async def _entity(self, issue) -> Entity:
"""Create an entity from an issue."""
return Entity(
key=issue["key"],
url=await self.__issue_landing_url(issue["key"]),
message=issue["message"],
severity=issue.get("severity", "no severity").lower(),
type=issue["type"].lower(),
component=issue["component"])
def __card_to_entity(card, api_url: URL, board_slug: str,
list_title: str) -> Entity:
"""Convert a card into a entity."""
return Entity(
key=card["_id"],
url=f"{api_url}/b/{card['boardId']}/{board_slug}/{card['_id']}",
list=list_title,
title=card["title"],
due_date=card.get("dueAt", ""),
date_last_activity=card["dateLastActivity"])
def _parse_entity( # pylint: disable=no-self-use
self, dependency: Element, dependency_index: int, namespaces: Namespaces, landing_url: str) -> Entity:
"""Parse the entity from the dependency."""
file_path = dependency.findtext("ns:filePath", default="", namespaces=namespaces)
sha1 = dependency.findtext("ns:sha1", namespaces=namespaces)
# We can only generate an entity landing url if a sha1 is present in the XML, but unfortunately not all
# dependencies have one, so check for it:
entity_landing_url = f"{landing_url}#l{dependency_index + 1}_{sha1}" if sha1 else ""
key = sha1 if sha1 else sha1_hash(file_path)
return Entity(key=key, file_path=file_path, url=entity_landing_url)
async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
"""Return a list of warnings."""
entities = []
for response in responses:
entities.extend(
[Entity(
key=warning[self.KEY], package=warning[self.PACKAGE], installed=warning[self.INSTALLED],
affected=warning[self.AFFECTED], vulnerability=warning[self.VULNERABILITY])
for warning in await response.json(content_type=None)])
return SourceMeasurement(entities=entities)
async def _entities(self, metrics: Dict[str, str]) -> List[Entity]:
entities = []
api_values = self._data_model["sources"][self.source_type]["parameters"]["effort_types"]["api_values"]
for effort_type in self.__effort_types():
effort_type_description = [param for param, api_key in api_values.items() if effort_type == api_key][0]
entities.append(
Entity(
key=effort_type, effort_type=effort_type_description, effort=metrics[effort_type],
url=await self.__effort_type_landing_url(effort_type)))
return entities
def __entity(sprint: Sprint, sprint_points: SprintPoints, velocity_type: str, entity_url: URL) -> Entity:
"""Create a sprint entity."""
sprint_id = str(sprint["id"])
committed = sprint_points["estimated"]["text"]
completed = sprint_points["completed"]["text"]
difference = str(float(sprint_points["completed"]["value"] - sprint_points["estimated"]["value"]))
measured = dict(completed=completed, estimated=committed, difference=difference)[velocity_type]
return Entity(
key=sprint["id"], name=sprint["name"], goal=sprint.get("goal") or "", points_completed=completed,
points_committed=committed, points_measured=measured, points_difference=difference,
url=str(entity_url) + sprint_id)
async def _parse_entities(self, responses: SourceResponses) -> Entities:
"""Override to parse the work items from the WIQL query response."""
return Entities(
Entity(
key=work_item["id"],
project=work_item["fields"]["System.TeamProject"],
title=work_item["fields"]["System.Title"],
work_item_type=work_item["fields"]["System.WorkItemType"],
state=work_item["fields"]["System.State"],
url=work_item["url"],
) for work_item in await self._work_items(responses))
async def _entities(self, metrics: dict[str, str]) -> Entities:
"""Extend to return ncloc per language, if the users picked ncloc to measure."""
if self._value_key() == "ncloc":
# Our user picked non-commented lines of code (ncloc), so we can show the ncloc per language, skipping
# languages the user wants to ignore
return Entities(
Entity(key=language,
language=self.LANGUAGES.get(language, language),
ncloc=ncloc)
for language, ncloc in self.__language_ncloc(metrics))
return await super()._entities(metrics)
