def resolve_depencies(modules_ordered_dict):
""" Resolve depencies """
# Prepare module name map
module_name_map = dict()
for item in modules_ordered_dict:
try:
module_name_map[modules_ordered_dict[item].__class__.__module__.split(".")[-2]] = \
modules_ordered_dict[item]
except IndexError:
module_name_map[modules_ordered_dict[item].__module__.split(".")[-2]] = \
modules_ordered_dict[item]
# Check required depencies
for module_name in module_name_map:
for dependency in module_name_map[module_name].depends_on():
if dependency not in module_name_map:
log.error("Dependency %s not present (required by %s)", dependency, module_name)
raise RuntimeError("Required dependency not present")
# Walk modules
module_order = list()
visited_modules = set()
for module_name in module_name_map:
if module_name not in module_order:
_walk_module_depencies(module_name, module_name_map, module_order, visited_modules)
# Re-order modules
for module_name in module_order:
modules_ordered_dict.move_to_end(module_name_map[module_name].get_name())
def list_suites(self, config_seed, config_variable, config_file):
""" List available suites from config """
config = self._load_config(config_seed, config_variable, config_file)
if "suites" not in config:
log.error("Suites are not defined")
return list()
return list(config["suites"])
def prepare_jira_mapping(jira_service):
""" Make Jira mapping (for projects that are using custom values) """
if not jira_service or not jira_service.valid:
return dict()
jira_service.connect()
issue_type = "Bug"
if jira_service.fields["issuetype"]["name"] != "!default_issuetype":
issue_type = jira_service.fields["issuetype"]["name"]
project_priorities = get_project_priorities(jira_service.client,
jira_service.project,
issue_type)
if not project_priorities:
jira_service.client.close()
return dict()
logging.debug("%s %s priorities: %s", jira_service.project, issue_type,
str(project_priorities))
c = const
mapping = dict()
for severity in c.JIRA_SEVERITIES:
if severity not in project_priorities:
for alternative in c.JIRA_ALTERNATIVES[severity]:
if alternative in project_priorities:
logging.warning("Mapping %s %s Jira priority: %s -> %s",
jira_service.project, issue_type, severity,
alternative)
mapping[severity] = alternative
break
if severity not in mapping:
mapping[severity] = project_priorities[0]
logging.error(
"Failed to find Jira mapping for %s, using %s as a fallback",
severity, mapping[severity])
jira_service.client.close()
return mapping
def validate_config(config):
""" Validate config """
required = ["server", "login", "password", "mail_to"]
not_set = [item for item in required if item not in config]
if not_set:
error = f"Required configuration options not set: {', '.join(not_set)}"
log.error(error)
raise ValueError(error)
def validate_config(config):
""" Validate config """
required = ["rp_project_name", "rp_launch_name", "rp_host", "rp_token"]
not_set = [item for item in required if item not in config]
if not_set:
error = f"Required configuration options not set: {', '.join(not_set)}"
log.error(error)
raise ValueError(error)
def validate_config(config):
""" Validate config """
required = ["code"]
not_set = [item for item in required if item not in config]
if not_set:
error = f"Required configuration options not set: {', '.join(not_set)}"
log.error(error)
raise ValueError(error)
def validate_config(config):
""" Validate config """
required = ["url", "username", "password", "project"]
not_set = [item for item in required if item not in config]
if not_set:
error = f"Required configuration options not set: {', '.join(not_set)}"
log.error(error)
raise ValueError(error)
def __init__(self, url, user, password, project, fields=None):
self.valid = True
self.url = url
self.password = password
self.user = user
try:
self.connect()
except:
logging.error("Failed to connect to Jira")
self.valid = False
return
self.projects = [project.key for project in self.client.projects()]
self.project = project.upper()
if self.project not in self.projects:
# self.client.close()
logging.warning("Requested project not found in Jira projects")
# self.valid = False
# return
self.fields = {}
self.watchers = []
if isinstance(fields, dict):
if 'watchers' in fields.keys():
self.watchers = [item.strip() for item in fields.pop('watchers').split(",")]
all_jira_fields = self.client.fields()
for key, value in fields.items():
if value:
if isinstance(value, str) and const.JIRA_FIELD_DO_NOT_USE_VALUE in value:
self.fields[key] = value
continue
jira_keys = [item for item in all_jira_fields if item["id"] == key]
if not jira_keys:
jira_keys = [item for item in all_jira_fields
if item["name"].lower() == key.lower().replace('_', ' ')]
if len(jira_keys) == 1:
jira_key = jira_keys[0]
key_type = jira_key['schema']['type']
else:
logging.warning(f'Cannot recognize field {key}. This field will not be used.')
continue
if key_type in ['string', 'number', 'any'] or isinstance(value, dict):
_value = value
elif key_type == 'array':
if isinstance(value, str):
_value = [item.strip() for item in value.split(",")]
elif isinstance(value, int):
_value = [value]
else:
_value = value
else:
_value = {'name': value}
self.fields[jira_key['id']] = _value
if not self.fields.get('issuetype', None):
self.fields['issuetype'] = {'name': '!default_issuetype'}
self.client.close()
self.created_jira_tickets = list()
def execute(self, args):
""" Run the command """
log.debug("Starting")
if args.call_from_legacy:
log.warning("Called from legacy entry point")
# Init context
context = RunContext(args)
config = ConfigModel(context)
if args.list_suites or not args.suite:
suites = config.list_suites(args.config_seed, args.config_variable,
args.config_file)
if not args.suite:
log.error("Suite is not defined. Use --help to get help")
log.info("Available suites: %s", ", ".join(suites))
return
# Make instances
scanning = ScanningPerformer(context)
processing = ProcessingPerformer(context)
reporting = ReportingPerformer(context)
# Add to context
context.performers["scanning"] = scanning
context.performers["processing"] = processing
context.performers["reporting"] = reporting
# Init config
config.load(args.config_seed, args.config_variable, args.config_file,
args.suite)
scanning.validate_config(context.config)
processing.validate_config(context.config)
reporting.validate_config(context.config)
# Add meta to context
self._fill_context_meta(context)
# Load state
context.state.load()
# Prepare reporters first
reporting.prepare()
# Run actions
actions.run(context)
# Prepare scanning and processing
scanning.prepare()
processing.prepare()
# Perform
scanning.perform()
processing.perform()
reporting.perform()
# Done
context.state.save()
reporting.flush()
log.debug("Done")
# Show quality gate statistics if any
for line in context.get_meta("quality_gate_stats", list()):
log.info(line)
# Fail quality gate if needed
if context.get_meta("fail_quality_gate", False):
os._exit(1) # pylint: disable=W0212
def validate_config(config):
""" Validate config """
required = [
"qualys_api_server", "qualys_login", "qualys_password",
"qualys_option_profile_id", "qualys_report_template_id",
"qualys_scanner_type", "target"
]
not_set = [item for item in required if item not in config]
if not_set:
error = f"Required configuration options not set: {', '.join(not_set)}"
log.error(error)
raise ValueError(error)
def execute(self):
""" Run the scanner """
# Check if we are running inside SAST container
if njsscan is None:
log.error("NodeJsScan is not installed in this environment")
return
# Replace print function to hide njsscan print()s
original_print = print
builtins.print = lambda *args, **kwargs: log.debug(" ".join(
[str(item) for item in args]))
try:
# Prepare excludes
excludes = self.config.get("excludes", list())
if not isinstance(excludes, list):
excludes = [item.strip() for item in excludes.split(",")]
log.debug("Excludes: %s", excludes)
# Collect files to scan
scan_target = list()
base = os.path.normpath(self.config.get("code"))
for root, _, files in os.walk(base):
# Normalize relative dir path
subpath = os.path.normpath(root)[len(base):]
if subpath.startswith(os.sep):
subpath = subpath[len(os.sep):]
# Check if dir (or any parent) is in excludes
skip_dir = False
for item in excludes:
if item.endswith(os.sep) and subpath.startswith(item):
skip_dir = True
# Skip dir if needed
if subpath + os.sep in excludes or skip_dir:
log.debug("Skipping dir %s", root)
continue
# Iterate files
for name in files:
target = os.path.join(root, name)
# Skip file if in excludes (direct match)
if os.path.join(subpath, name) in excludes:
log.debug("Skipping file %s", target)
continue
# Add to files to scan
scan_target.append(target)
# Run scanner
result = njsscan.scan_file(scan_target)
finally:
# Restore print function
builtins.print = original_print
# Parse result
parse_findings(result, self)
# Save intermediates
self.save_intermediates(result)
def execute(self):
""" Run the scanner """
try:
self._start_zap()
if not self._wait_for_zap_start():
log.error("ZAP failed to start")
error = Error(
tool=self.get_name(),
error="ZAP failed to start",
details="ZAP daemon failed to start"
)
self.errors.append(error)
return
log.info("Target: %s", self.config.get("target"))
self._prepare_context()
self._setup_scan_policy()
self._spider()
self._wait_for_passive_scan()
self._ajax_spider()
self._wait_for_passive_scan()
self._active_scan()
self._wait_for_passive_scan()
except:
log.exception("Exception during ZAP scanning")
error = Error(
tool=self.get_name(),
error=f"Exception during ZAP scanning",
details=f"```\n{traceback.format_exc()}\n```"
)
self.errors.append(error)
finally:
try:
# Get report
log.info("Getting ZAP report")
zap_report = self._zap_api.core.jsonreport()
# Parse JSON
log.info("Processing findings")
parse_findings(zap_report, self)
except:
log.exception("Exception during ZAP findings processing")
error = Error(
tool=self.get_name(),
error=f"Exception during ZAP findings processing",
details=f"```\n{traceback.format_exc()}\n```"
)
self.errors.append(error)
self._save_intermediates()
pkg_resources.cleanup_resources()
self._stop_zap()
def execute(self):
""" Run the scanner """
log.debug(f"Config: {self.config}")
try:
self._start_zap()
if not self._wait_for_zap_start():
log.error("ZAP failed to start")
self.errors.append("ZAP daemon failed to start")
return
log.info("Target: %s", self.config.get("target"))
except BaseException as exception:
log.exception("Exception during ZAP scanning")
self.errors.append(str(exception))
finally:
self._stop_zap()
def _walk_module_depencies(module_name, module_name_map, module_order, visited_modules):
# Collect depencies
depencies = list()
depencies.extend(module_name_map[module_name].depends_on())
for optional_dependency in module_name_map[module_name].run_after():
if optional_dependency in module_name_map:
depencies.append(optional_dependency)
# Resolve
visited_modules.add(module_name)
for dependency in depencies:
if dependency not in module_order:
if dependency in visited_modules:
log.error("Circular dependency (%s <-> %s)", dependency, module_name)
raise RuntimeError("Circular dependency present")
_walk_module_depencies(dependency, module_name_map, module_order, visited_modules)
# Add to resolved order
module_order.append(module_name)
def _create_vault_client(self):
client = hvac.Client(url=self.config["url"],
verify=self.config.get("ssl_verify", False),
namespace=self.config.get("namespace", None))
if "auth_token" in self.config:
client.token = self.config["auth_token"]
if "auth_username" in self.config:
client.auth_userpass(self.config.get("auth_username"),
self.config.get("auth_password", ""))
if "auth_role_id" in self.config:
client.auth_approle(self.config.get("auth_role_id"),
self.config.get("auth_secret_id", ""))
if not client.is_authenticated():
error = "Vault authentication failed"
log.error(error)
raise ValueError(error)
return client
def get_or_create_issue(self, search_string, issue_data):
issuetype = issue_data['issuetype']
created = False
jira_results = self.client.search_issues(search_string)
issues = []
for each in jira_results:
if each.fields.summary == issue_data.get('summary', None):
issues.append(each)
if len(issues) == 1:
issue = issues[0]
if len(issues) > 1:
logging.error(' more then 1 issue with the same summary')
else:
logging.info(f' {issuetype["name"]} already exists: {issue.key}')
else:
issue = self.post_issue(issue_data)
created = True
return issue, created
def get_project_priorities(jira_client, project, issue_type="Bug"):
""" Returns list of Jira priorities in project """
try:
meta = jira_client.createmeta(projectKeys=project,
issuetypeNames=issue_type,
expand="projects.issuetypes.fields")
logging.debug("Got metadata for %d projects", len(meta["projects"]))
if not meta["projects"]:
logging.error("No meta returned for %s with type %s", project,
issue_type)
return []
project_meta = meta["projects"][0]
logging.debug("Got metadata for %d issuetypes",
len(project_meta["issuetypes"]))
if not project_meta["issuetypes"]:
logging.error("No %s in %s", issue_type, project)
return []
issue_types = project_meta["issuetypes"][0]
if "priority" not in issue_types["fields"]:
logging.error("No priority field in %s", project)
return []
priorities = [
priority["name"] for priority in \
issue_types["fields"]["priority"]["allowedValues"]
]
return priorities
except: # pylint: disable=W0702
logging.exception("Failed to get meta for %s", project)
return []
def report(self):
""" Report """
log.info("Sending jUnit report to Galloper")
# Get options
bucket = self.config.get("bucket")
tgtobj = self.config.get("object")
# Get jUnit report path
junit_report_file = \
self.context.performers["reporting"].get_module_meta("junit", "report_file", None)
if not junit_report_file:
log.error("jUnit report not present")
return
# Send to Galloper
with open(junit_report_file, "rb") as tgt_file:
headers = dict()
if os.environ.get("token"):
headers["Authorization"] = f"Bearer {os.environ.get('token')}"
url = f"{os.environ.get('galloper_url')}/api/v1/artifacts/" \
f"{os.environ.get('project_id')}/{bucket}/{tgtobj}"
requests.post(
url, headers=headers, files={
"file": (f"{tgtobj}", tgt_file)
}
)
def _validate_config_base(self, config):
if config.get(constants.CONFIG_VERSION_KEY,
0) != constants.CURRENT_CONFIG_VERSION:
log.error("Invalid config version")
return False
if "suites" not in config:
log.error("Suites are not defined")
return False
if not config["suites"].get(self.context.suite, None):
log.error("Suite is not defined: %s", self.context.suite)
return False
return True
def _validate_config_base(self, config, suite):
if config.get(constants.CONFIG_VERSION_KEY, 0) != constants.CURRENT_CONFIG_VERSION:
log.error("Invalid config version")
return False
if "global" not in config:
config["global"] = dict()
if "suites" not in config:
log.error("Suites are not defined")
return False
if not config["suites"].get(suite, None):
log.error("Suite is not defined: %s", suite)
log.info("Available suites: %s", ", ".join(list(config["suites"])))
return False
if "settings" not in config["suites"][suite]:
config["suites"][suite]["settings"] = dict()
return True
def validate_config(config):
""" Validate config """
if "reporters" not in config:
log.error("No reporters defined in config")
raise ValueError("No reporters configuration present")
def validate_config(config):
""" Validate config """
log.debug(f"Config: {config}")
if "target" not in config:
log.error("No target defined in config")
raise ValueError("No target configuration present")
def parse_findings(output_file, scanner):
""" Parse findings (code from dusty 1.0) """
log.debug("Parsing findings")
nscan = parse(output_file)
root = nscan.getroot()
# Check validity
if "nmaprun" not in root.tag:
log.error(
"Exception during Nmap findings processing: invalid XML file")
error = Error(
tool=scanner.get_name(),
error=f"Exception during Nmap findings processing",
details=f"Output file doesn't seem to be a valid Nmap xml file.")
scanner.errors.append(error)
return
dupes = dict()
hostInfo = ""
for host in root.iter("host"):
ip = host.find("address[@addrtype='ipv4']").attrib["addr"]
fqdn = None
if host.find("hostnames/hostname[@type='PTR']") is not None:
fqdn = host.find("hostnames/hostname[@type='PTR']").attrib["name"]
#
for os in root.iter("os"):
if ip is not None:
hostInfo += "IP Address: %s\n" % ip
if fqdn is not None:
fqdn += "FQDN: %s\n" % ip
for osv in os.iter("osmatch"):
if "name" in osv.attrib:
hostInfo += "Host OS: %s\n" % osv.attrib["name"]
if "accuracy" in osv.attrib:
hostInfo += "Accuracy: {0}%\n".format(
osv.attrib["accuracy"])
hostInfo += "\n"
#
xpath_port_selector = "ports/port[state/@state='open']"
if scanner.config.get("include_unfiltered", False):
xpath_port_selector = "ports/port[state/@state=('open','unfiltered')]"
#
for portelem in elementpath.select(host, xpath_port_selector):
port = portelem.attrib["portid"]
protocol = portelem.attrib["protocol"]
#
title = f"Open port: {ip}:{port}/{protocol}"
description = hostInfo
description += f"Port: {port}\n"
serviceinfo = ""
#
if portelem.find("service") is not None:
if "product" in portelem.find("service").attrib:
serviceinfo += "Product: %s\n" % portelem.find(
"service").attrib["product"]
#
if "version" in portelem.find("service").attrib:
serviceinfo += "Version: %s\n" % portelem.find(
"service").attrib["version"]
#
if "extrainfo" in portelem.find("service").attrib:
serviceinfo += "Extra Info: %s\n" % portelem.find(
"service").attrib["extrainfo"]
#
description += serviceinfo
#
description += "\n\n"
#
dupe_key = f"{port}_{protocol}_{ip}"
if dupe_key in dupes:
find = dupes[dupe_key]
if description is not None:
find["description"] += description
else:
find = {
"title": title,
"description": description,
"endpoints": list()
}
find["endpoints"].append(f"{ip}:{port}/{protocol}")
dupes[dupe_key] = find
# Create finding objects
for item in dupes.values():
finding = DastFinding(title=item["title"],
description=markdown.markdown_escape(
item["description"]))
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", SEVERITIES[-1])
# Endpoints (for backwards compatibility)
endpoints = list()
for entry in item["endpoints"]:
endpoint = url.parse_url(entry)
if endpoint in endpoints:
continue
endpoints.append(endpoint)
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
# Done
scanner.findings.append(finding)
def report_multi(self):
""" Report """
wrappers_config = dict()
wrappers_config[None] = self.config
for key, value in self.config.get("dynamic_jira").items():
wrappers_config[re.compile(key)] = value
#
wrappers = dict()
for wrapper_key, wrapper_config in wrappers_config.items():
wrapper = JiraWrapper(
wrapper_config.get("url"),
wrapper_config.get("username"),
wrapper_config.get("password"),
wrapper_config.get("project"),
wrapper_config.get("fields"),
)
#
if not wrapper.valid:
# Save default mapping to meta as a fallback
default_mapping = constants.JIRA_SEVERITY_MAPPING
default_mapping.update(
self.config.get("custom_mapping", dict()))
self.set_meta("mapping", default_mapping)
# Report error
log.error(
"Jira configuration is invalid. Skipping Jira reporting")
raise RuntimeError("Jira configuration is invalid")
#
wrappers[wrapper_key] = dict()
wrappers[wrapper_key]["wrapper"] = wrapper
wrappers[wrapper_key]["config"] = wrapper_config
#
if wrapper_config.get("separate_epic_linkage", False) and \
"Epic Link" in wrapper_config.get("fields"):
wrappers[wrapper_key]["epic_link"] = wrapper_config.get(
"fields").pop("Epic Link")
#
wrappers[wrapper_key]["raw_epic_link"] = None
if wrapper_config.get("separate_epic_linkage", False):
wrappers[wrapper_key]["raw_epic_link"] = wrappers[wrapper_key][
"epic_link"]
elif "Epic Link" in wrapper_config.get("fields"):
wrappers[wrapper_key]["raw_epic_link"] = wrapper_config.get(
"fields")["Epic Link"]
#
wrappers[wrapper_key]["priority_mapping"] = wrapper_config.get(
"custom_mapping", prepare_jira_mapping(wrapper))
wrappers[wrapper_key]["mapping_meta"] = dict(
wrappers[wrapper_key]["priority_mapping"])
#
self.set_meta("wrapper", wrappers[None]["wrapper"])
self.set_meta("raw_epic_link", wrappers[None]["raw_epic_link"])
#
dynamic_label_mapping = dict()
if self.config.get("dynamic_labels", None):
try:
for key, value in self.config.get("dynamic_labels").items():
dynamic_label_mapping[re.compile(key)] = value
except: # pylint: disable=W0702
log.exception("Failed to add dynamic label mapping")
#
dynamic_field_mapping = dict()
if self.config.get("dynamic_fields", None):
try:
for key, value in self.config.get("dynamic_fields").items():
dynamic_field_mapping[re.compile(key)] = value
except: # pylint: disable=W0702
log.exception("Failed to add dynamic field mapping")
#
findings = list()
for item in self.context.findings: # pylint: disable=R1702
#
if item.get_meta("information_finding", False) or \
item.get_meta("false_positive_finding", False) or \
item.get_meta("excluded_finding", False):
continue
#
if isinstance(item, (DastFinding, SastFinding)):
#
dynamic_labels = list()
dynamic_fields = list()
dynamic_wrapper = wrappers[None]
#
for endpoint in item.get_meta("endpoints", list()):
#
for pattern, addon_label in dynamic_label_mapping.items():
try:
if pattern.match(endpoint.raw):
dynamic_labels.append(addon_label)
except: # pylint: disable=W0702
log.exception("Failed to add dynamic label")
#
for pattern, addon_fields in dynamic_field_mapping.items():
try:
if pattern.match(endpoint.raw):
dynamic_fields.append(addon_fields)
except: # pylint: disable=W0702
log.exception("Failed to add dynamic field")
#
for pattern, addon_jira in wrappers.items():
if pattern is None:
continue
try:
if pattern.match(endpoint.raw):
dynamic_wrapper = addon_jira
except: # pylint: disable=W0702
log.exception("Failed to add dynamic JIRA")
#
severity = item.get_meta("severity", SEVERITIES[-1])
priority = constants.JIRA_SEVERITY_MAPPING[severity]
if dynamic_wrapper["priority_mapping"] and \
priority in dynamic_wrapper["priority_mapping"]:
priority = dynamic_wrapper["priority_mapping"][priority]
dynamic_wrapper["mapping_meta"][severity] = priority
#
if isinstance(item, DastFinding):
findings.append({
"title":
item.title,
"priority":
priority,
"description":
item.description.replace("\\.", "."),
"issue_hash":
item.get_meta("issue_hash", "<no_hash>"),
"additional_labels": [
label.replace(" ", "_") for label in [
item.get_meta("tool", "scanner"),
self.context.get_meta("testing_type", "DAST"),
item.get_meta("severity", SEVERITIES[-1])
]
] + dynamic_labels,
"dynamic_fields":
dynamic_fields,
"raw":
item,
"wrapper":
dynamic_wrapper,
})
elif isinstance(item, SastFinding):
#
description_chunks = [
item.replace("\\.", ".").replace(
"<pre>", "{code:collapse=true}\n\n").replace(
"</pre>",
"\n\n{code}").replace("<br />", "\n")
for item in item.description
]
#
if len("\n\n".join(description_chunks)
) > constants.JIRA_DESCRIPTION_MAX_SIZE:
description = description_chunks[0]
chunks = description_chunks[1:]
comments = list()
new_line_str = ' \n \n'
for chunk in chunks:
if not comments or (
len(comments[-1]) + len(new_line_str) + len(chunk)
) >= \
constants.JIRA_COMMENT_MAX_SIZE:
comments.append(cut_jira_comment(chunk))
else: # Last comment can handle one more chunk
comments[
-1] += new_line_str + cut_jira_comment(
chunk)
else:
description = "\n\n".join(description_chunks)
comments = list()
#
findings.append({
"title":
item.title,
"priority":
priority,
"description":
description,
"issue_hash":
item.get_meta("issue_hash", "<no_hash>"),
"additional_labels": [
label.replace(" ", "_") for label in [
item.get_meta("tool", "scanner"),
self.context.get_meta("testing_type", "SAST"),
item.get_meta("severity", SEVERITIES[-1])
]
] + dynamic_labels,
"dynamic_fields":
dynamic_fields,
"comments":
comments,
"raw":
item,
"wrapper":
dynamic_wrapper,
})
#
#
else:
log.warning("Unsupported finding type")
continue # raise ValueError("Unsupported item type")
#
self.set_meta("mapping", wrappers[None]["mapping_meta"])
#
for finding in findings:
if finding["wrapper"]["config"].get("max_description_size", False):
if len(finding["description"]) > \
int(finding["wrapper"]["config"].get("max_description_size")):
if "comments" not in finding:
finding["comments"] = list()
#
comment_chunks = list()
cut_line_len = len(constants.JIRA_DESCRIPTION_CUT)
cut_point = int(finding["wrapper"]["config"].get(
"max_description_size")) - cut_line_len
#
item_description = finding["description"]
finding["description"] = \
f"{item_description[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}"
#
description_data = item_description[cut_point:]
comment_cut_threshold = min(
constants.JIRA_COMMENT_MAX_SIZE,
int(finding["wrapper"]["config"].get(
"max_description_size")))
cut_point = comment_cut_threshold - cut_line_len
#
while description_data:
if len(description_data) > comment_cut_threshold:
comment_chunks.append(
f"{description_data[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}"
)
description_data = description_data[cut_point:]
else:
comment_chunks.append(description_data)
break
#
while comment_chunks:
finding["comments"].insert(0, comment_chunks.pop())
#
findings.sort(key=lambda item: (SEVERITIES.index(item["raw"].get_meta(
"severity", SEVERITIES[-1])), item["raw"].get_meta("tool", ""),
item["raw"].title))
#
new_tickets = list()
existing_tickets = list()
#
for _, local_wrapper in wrappers.items():
local_wrapper["wrapper"].connect()
#
for finding in findings:
try:
config_labels = finding["wrapper"]["config"].get(
"additional_labels", None)
if config_labels is None:
config_labels = list()
if not isinstance(config_labels, list):
config_labels = [
item.strip() for item in config_labels.split(",")
]
#
field_overrides = dict()
for dynamic_field in finding["dynamic_fields"]:
field_overrides.update(dynamic_field)
#
issue, created = finding["wrapper"]["wrapper"].create_issue(
finding["title"], # title
finding["priority"], # priority
finding["description"], # description
finding["issue_hash"], # issue_hash, self.get_hash_code()
# attachments=None,
# get_or_create=True,
additional_labels=finding["additional_labels"] +
config_labels, # additional_labels # pylint: disable=C0301
field_overrides=field_overrides,
)
if created and "comments" in finding:
for comment in finding["comments"]:
finding["wrapper"]["wrapper"].add_comment_to_issue(
issue, comment)
if created and finding["wrapper"]["config"].get(
"separate_epic_linkage", False):
try:
finding["wrapper"][
"wrapper"].client.add_issues_to_epic(
finding["wrapper"]["epic_link"],
[str(issue.key)])
except: # pylint: disable=W0702
log.exception("Failed to add ticket %s to epic %s",
str(issue.key),
finding["wrapper"]["epic_link"])
try:
result_priority = str(issue.fields.priority)
except: # pylint: disable=W0702
result_priority = "Default"
#
ticket_meta = {
"jira_id":
issue.key,
"jira_url":
f"{finding['wrapper']['config'].get('url')}/browse/{issue.key}", # pylint: disable=C0301
"priority":
result_priority,
"status":
issue.fields.status.name,
"created":
issue.fields.created,
"open_date":
datetime.strptime(
issue.fields.created,
"%Y-%m-%dT%H:%M:%S.%f%z").strftime("%d %b %Y %H:%M"),
"description":
issue.fields.summary,
"assignee":
str(issue.fields.assignee),
"raw_created":
str(issue.fields.created),
"raw_severity":
finding["raw"].get_meta("severity", SEVERITIES[-1]),
"raw_jira_url":
finding["wrapper"]["config"].get("url"),
"raw_jira_project":
finding["wrapper"]["config"].get("project"),
"raw_jira_epic":
finding["wrapper"]["raw_epic_link"],
"raw_jira_fields":
finding["wrapper"]["config"].get("fields"),
"raw_addon_fields":
field_overrides,
"raw_addon_labels":
finding["additional_labels"] + config_labels,
}
if created:
if not self._ticket_in_list(ticket_meta, new_tickets):
new_tickets.append(ticket_meta)
else:
if issue.fields.status.name in constants.JIRA_OPENED_STATUSES:
if not self._ticket_in_list(ticket_meta,
existing_tickets):
existing_tickets.append(ticket_meta)
except: # pylint: disable=W0702
log.exception(
f"Failed to create ticket for {finding['title']}")
error = Error(
tool=self.get_name(),
error=f"Failed to create ticket for {finding['title']}",
details=f"```\n{traceback.format_exc()}\n```")
self.errors.append(error)
#
self.set_meta("new_tickets", new_tickets)
self.set_meta("existing_tickets", existing_tickets)
def interrupt_handler(signal_, frame): # pylint: disable=W0613
""" Handle interrupt signals """
log.error("Got interrupt signal. Terminating")
os._exit(1) # pylint: disable=W0212
def run(self):
""" Run action """
# Patch dulwich to work without valid UID/GID
dulwich.repo.__original__get_default_identity = dulwich.repo._get_default_identity # pylint: disable=W0212
dulwich.repo._get_default_identity = _dulwich_repo_get_default_identity # pylint: disable=W0212
# Patch dulwich to use paramiko SSH client
dulwich.client.get_ssh_vendor = ParamikoSSHVendor
# Patch paramiko to skip key verification
paramiko.transport.Transport._verify_key = _paramiko_transport_verify_key # pylint: disable=W0212
# Set USERNAME if needed
try:
getpass.getuser()
except: # pylint: disable=W0702
os.environ["USERNAME"] = "******"
# Get options
source = self.config.get("source")
target = self.config.get("target")
branch = self.config.get("branch", "master")
depth = self.config.get("depth", None)
# Prepare auth
auth_args = dict()
if self.config.get("username", None) is not None:
auth_args["username"] = self.config.get("username")
if self.config.get("password", None) is not None:
auth_args["password"] = self.config.get("password")
if self.config.get("key", None) is not None:
auth_args["key_filename"] = self.config.get("key")
if self.config.get("key_data", None) is not None:
key_obj = io.StringIO(
self.config.get("key_data").replace("|", "\n"))
pkey = paramiko.RSAKey.from_private_key(key_obj)
# Patch paramiko to use our key
paramiko.client.SSHClient._auth = _paramiko_client_SSHClient_auth( # pylint: disable=W0212
paramiko.client.SSHClient._auth,
pkey # pylint: disable=W0212
)
# Clone repository
log.info("Cloning repository %s into %s", source, target)
repository = porcelain.clone(source,
target,
checkout=False,
depth=depth,
errstream=log.DebugLogStream(),
**auth_args)
# Get current HEAD tree (default branch)
try:
head_tree = repository[b"HEAD"]
except: # pylint: disable=W0702
head_tree = None
# Get target tree (requested branch)
branch_b = branch.encode("utf-8")
try:
target_tree = repository[b"refs/remotes/origin/" + branch_b]
except: # pylint: disable=W0702
target_tree = None
# Checkout branch
if target_tree is not None:
log.info("Checking out branch %s", branch)
repository[b"refs/heads/" +
branch_b] = repository[b"refs/remotes/origin/" +
branch_b]
repository.refs.set_symbolic_ref(b"HEAD",
b"refs/heads/" + branch_b)
repository.reset_index(repository[b"HEAD"].tree)
elif head_tree is not None:
try:
default_branch_name = repository.refs.follow(b"HEAD")[0][1]
if default_branch_name.startswith(refs.LOCAL_BRANCH_PREFIX):
default_branch_name = default_branch_name[
len(refs.LOCAL_BRANCH_PREFIX):]
default_branch_name = default_branch_name.decode("utf-8")
log.warning(
"Branch %s was not found. Checking out default branch %s",
branch, default_branch_name)
except: # pylint: disable=W0702
log.warning(
"Branch %s was not found. Trying to check out default branch",
branch)
try:
repository.reset_index(repository[b"HEAD"].tree)
except: # pylint: disable=W0702
log.exception("Failed to checkout default branch")
else:
log.error(
"Branch %s was not found and default branch is not set. Skipping checkout"
)
# Delete .git if requested
if self.config.get("delete_git_dir", False):
log.info("Deleting .git directory")
shutil.rmtree(os.path.join(target, ".git"))
def execute(self, args):
""" Run the command """
log.debug("Starting")
# Check args
if not args.source or not args.target:
log.error("Please specify source and target.")
return
# Patch dulwich to work without valid UID/GID
dulwich.repo.__original__get_default_identity = dulwich.repo._get_default_identity # pylint: disable=W0212
dulwich.repo._get_default_identity = _dulwich_repo_get_default_identity # pylint: disable=W0212
# Patch dulwich to use paramiko SSH client
dulwich.client.get_ssh_vendor = ParamikoSSHVendor
# Patch paramiko to skip key verification
paramiko.transport.Transport._verify_key = _paramiko_transport_verify_key # pylint: disable=W0212
# Set USERNAME if needed
try:
getpass.getuser()
except: # pylint: disable=W0702
os.environ["USERNAME"] = "******"
# Fill args
depth = None
if args.depth:
depth = args.depth
# Prepare auth
auth_args = dict()
# Take from env variables
if args.username_variable and args.username_variable in os.environ:
auth_args["username"] = os.environ[args.username_variable]
os.environ["USERNAME"] = os.environ[args.username_variable]
if args.password_variable and args.password_variable in os.environ:
auth_args["password"] = os.environ[args.password_variable]
if args.key_variable and args.key_variable in os.environ:
auth_args["key_filename"] = os.environ[args.key_variable]
if args.key_data_variable and args.key_data_variable in os.environ:
key_obj = io.StringIO(os.environ[args.key_data_variable].replace(
"|", "\n"))
pkey = paramiko.RSAKey.from_private_key(key_obj)
# Patch paramiko to use our key
paramiko.client.SSHClient._auth = _paramiko_client_SSHClient_auth( # pylint: disable=W0212
paramiko.client.SSHClient._auth,
pkey # pylint: disable=W0212
)
# Take from commandline parameters
if args.username:
auth_args["username"] = args.username
os.environ["USERNAME"] = args.username
if args.password:
auth_args["password"] = args.password
if args.key:
auth_args["key_filename"] = args.key
if args.key_data:
key_obj = io.StringIO(args.key_data.replace("|", "\n"))
pkey = paramiko.RSAKey.from_private_key(key_obj)
# Patch paramiko to use our key
paramiko.client.SSHClient._auth = _paramiko_client_SSHClient_auth( # pylint: disable=W0212
paramiko.client.SSHClient._auth,
pkey # pylint: disable=W0212
)
# Clone repository
log.info("Cloning repository %s into %s", args.source, args.target)
repository = porcelain.clone(args.source,
args.target,
checkout=False,
depth=depth,
errstream=log.DebugLogStream(),
**auth_args)
# Checkout branch
log.info("Checking out branch %s", args.branch)
branch = args.branch.encode("utf-8")
repository[b"refs/heads/" +
branch] = repository[b"refs/remotes/origin/" + branch]
repository.refs.set_symbolic_ref(b"HEAD", b"refs/heads/" + branch)
repository.reset_index(repository[b"HEAD"].tree)
def parse_findings(data, scanner):
""" Parse findings """
log.debug("Parsing findings")
parser = etree.XMLParser(remove_blank_text=True,
no_network=True,
recover=True)
obj = etree.fromstring(data, parser)
qids = obj.xpath("/WAS_WEBAPP_REPORT/GLOSSARY/QID_LIST/QID")
disabled_titles = constants.QUALYS_DISABLED_TITLES
for qid in qids:
qid_title = qid.findtext("TITLE")
if qid_title not in disabled_titles:
_qid = qid.findtext("QID")
qid_solution = qid.findtext("SOLUTION")
qid_description = qid.findtext("DESCRIPTION")
qid_impact = qid.findtext("IMPACT")
qid_category = qid.findtext("CATEGORY")
qid_severity = "Info"
owasp = qid.findtext("OWASP") if qid.findtext("OWASP") else ""
wasc = qid.findtext("WASC") if qid.findtext("WASC") else ""
cwe = qid.findtext("CWE") if qid.findtext("CWE") else ""
cvss_base = qid.findtext("CVSS_BASE") if qid.findtext(
"CVSS_BASE") else ""
if qid.xpath("SEVERITY"):
qid_severity = constants.QUALYS_SEVERITIES[int(
qid.findtext("SEVERITY"))]
references = []
entrypoints = []
if "Information Gathered" in qid_category:
qid_severity = "Info"
records = obj.xpath(
f'//INFORMATION_GATHERED_LIST/INFORMATION_GATHERED/QID[text()="{_qid}"]/..'
)
for record in records:
references.append(
html.escape(
base64.b64decode(record.findtext("DATA")).decode(
"utf-8", errors="ignore")))
else:
records = obj.xpath(
f'//VULNERABILITY_LIST/VULNERABILITY/QID[text()="{_qid}"]/..'
)
for record in records:
record_url = record.findtext('URL')
access_pass = [
a.text for a in records[0].xpath('ACCESS_PATH/URL')
]
method = record.findtext('PAYLOADS/PAYLOAD/REQUEST/METHOD')
if not method:
log.error("Bad record: %s", str(record))
method = ""
request = record.findtext('PAYLOADS/PAYLOAD/REQUEST/URL')
request = html.escape(request)
response = record.findtext(
'PAYLOADS/PAYLOAD/RESPONSE/CONTENTS')
response = html.escape(
base64.b64decode(response).decode("utf-8",
errors="ignore"))
entrypoints.append(record_url)
entrypoints.extend(access_pass)
references.append(
f"{method.upper()}: {request}\n\nResponse: {response}\n\n"
)
for reference in references:
description = f"{markdown.html_to_text(qid_description)}\n\n"
if qid_impact:
description += f"**Impact:**\n {markdown.html_to_text(qid_impact)}\n\n"
if qid_solution:
description += f"**Mitigation:**\n {markdown.html_to_text(qid_solution)}\n\n"
if reference:
description += f"**References:**\n {markdown.markdown_escape(reference)}\n\n"
if cwe:
description += f"**CWE:** {markdown.markdown_escape(cwe)}\n\n"
if owasp:
description += f"**OWASP:** {markdown.markdown_escape(owasp)}\n\n"
if wasc:
description += f"**WASC:** {markdown.markdown_escape(wasc)}\n\n"
if cvss_base:
description += f"**CVSS_BASE:** {markdown.markdown_escape(cvss_base)}\n\n"
# Make finding object
finding = DastFinding(title=f"{qid_title} - {qid_category}",
description=description)
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", qid_severity)
# Endpoints (for backwards compatibility)
endpoints = list()
for item in entrypoints:
endpoint = url.parse_url(item)
if endpoint in endpoints:
continue
endpoints.append(endpoint)
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
# Done
scanner.findings.append(finding)
def report(self):
""" Report """
# Remove "Epic Link" from fields if requested
if self.config.get("separate_epic_linkage", False) and \
"Epic Link" in self.config.get("fields"):
epic_link = self.config.get("fields").pop("Epic Link")
# Prepare wrapper
log.info("Creating legacy wrapper instance")
wrapper = JiraWrapper(self.config.get("url"),
self.config.get("username"),
self.config.get("password"),
self.config.get("project"),
self.config.get("fields"))
if not wrapper.valid:
# Save default mapping to meta as a fallback
default_mapping = constants.JIRA_SEVERITY_MAPPING
default_mapping.update(self.config.get("custom_mapping", dict()))
self.set_meta("mapping", default_mapping)
# Report error
log.error("Jira configuration is invalid. Skipping Jira reporting")
raise RuntimeError("Jira configuration is invalid")
log.debug("Legacy wrapper is valid")
# Prepare findings
priority_mapping = self.config.get("custom_mapping",
prepare_jira_mapping(wrapper))
mapping_meta = dict(priority_mapping)
findings = list()
for item in self.context.findings:
if item.get_meta("information_finding", False) or \
item.get_meta("false_positive_finding", False) or \
item.get_meta("excluded_finding", False):
continue
if isinstance(item, DastFinding):
severity = item.get_meta("severity", SEVERITIES[-1])
priority = constants.JIRA_SEVERITY_MAPPING[severity]
if priority_mapping and priority in priority_mapping:
priority = priority_mapping[priority]
mapping_meta[
severity] = priority # Update meta mapping to reflect actual results
findings.append({
"title":
item.title,
"priority":
priority,
"description":
item.description.replace("\\.", "."),
"issue_hash":
item.get_meta("issue_hash", "<no_hash>"),
"additional_labels": [
label.replace(" ", "_") for label in [
item.get_meta("tool", "scanner"),
self.context.get_meta("testing_type", "DAST"),
item.get_meta("severity", SEVERITIES[-1])
]
],
"raw":
item
})
elif isinstance(item, SastFinding):
severity = item.get_meta("severity", SEVERITIES[-1])
priority = constants.JIRA_SEVERITY_MAPPING[severity]
if priority_mapping and priority in priority_mapping:
priority = priority_mapping[priority]
mapping_meta[
severity] = priority # Update meta mapping to reflect actual results
description_chunks = [
item.replace("\\.", ".").replace(
"<pre>", "{code:collapse=true}\n\n").replace(
"</pre>", "\n\n{code}").replace("<br />", "\n")
for item in item.description
]
if len("\n\n".join(description_chunks)
) > constants.JIRA_DESCRIPTION_MAX_SIZE:
description = description_chunks[0]
chunks = description_chunks[1:]
comments = list()
new_line_str = ' \n \n'
for chunk in chunks:
if not comments or (len(comments[-1]) + len(new_line_str) + len(chunk)) >= \
constants.JIRA_COMMENT_MAX_SIZE:
comments.append(cut_jira_comment(chunk))
else: # Last comment can handle one more chunk
comments[-1] += new_line_str + cut_jira_comment(
chunk)
else:
description = "\n\n".join(description_chunks)
comments = list()
findings.append({
"title":
item.title,
"priority":
priority,
"description":
description,
"issue_hash":
item.get_meta("issue_hash", "<no_hash>"),
"additional_labels": [
label.replace(" ", "_") for label in [
item.get_meta("tool", "scanner"),
self.context.get_meta("testing_type", "SAST"),
item.get_meta("severity", SEVERITIES[-1])
]
],
"comments":
comments,
"raw":
item
})
else:
log.warning("Unsupported finding type")
continue # raise ValueError("Unsupported item type")
# Cut description if length above configured limit
if self.config.get("max_description_size", False):
for finding in findings:
if len(finding["description"]) > int(
self.config.get("max_description_size")):
if "comments" not in finding:
finding["comments"] = list()
#
comment_chunks = list()
cut_line_len = len(constants.JIRA_DESCRIPTION_CUT)
cut_point = int(
self.config.get("max_description_size")) - cut_line_len
#
item_description = finding["description"]
finding["description"] = \
f"{item_description[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}"
#
description_data = item_description[cut_point:]
comment_cut_threshold = min(
constants.JIRA_COMMENT_MAX_SIZE,
int(self.config.get("max_description_size")))
cut_point = comment_cut_threshold - cut_line_len
#
while description_data:
if len(description_data) > comment_cut_threshold:
comment_chunks.append(
f"{description_data[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}"
)
description_data = description_data[cut_point:]
else:
comment_chunks.append(description_data)
break
#
while comment_chunks:
finding["comments"].insert(0, comment_chunks.pop())
# Sort findings by severity-tool-title
findings.sort(key=lambda item: (SEVERITIES.index(item["raw"].get_meta(
"severity", SEVERITIES[-1])), item["raw"].get_meta("tool", ""),
item["raw"].title))
# Submit issues
wrapper.connect()
new_tickets = list()
existing_tickets = list()
for finding in findings:
try:
issue, created = wrapper.create_issue(
finding["title"], # title
finding["priority"], # priority
finding["description"], # description
finding["issue_hash"], # issue_hash, self.get_hash_code()
# attachments=None,
# get_or_create=True,
additional_labels=finding[
"additional_labels"] # additional_labels
)
if created and "comments" in finding:
for comment in finding["comments"]:
wrapper.add_comment_to_issue(issue, comment)
if created and self.config.get("separate_epic_linkage", False):
try:
wrapper.client.add_issues_to_epic(
epic_link, [str(issue.key)])
except: # pylint: disable=W0702
log.exception("Failed to add ticket %s to epic %s",
str(issue.key), epic_link)
try:
result_priority = issue.fields.priority
except: # pylint: disable=W0702
result_priority = "Default"
ticket_meta = {
"jira_id":
issue.key,
"jira_url":
f"{self.config.get('url')}/browse/{issue.key}",
"priority":
result_priority,
"status":
issue.fields.status.name,
"created":
issue.fields.created,
"open_date":
datetime.strptime(
issue.fields.created,
"%Y-%m-%dT%H:%M:%S.%f%z").strftime("%d %b %Y %H:%M"),
"description":
issue.fields.summary,
"assignee":
issue.fields.assignee
}
if created:
if not self._ticket_in_list(ticket_meta, new_tickets):
new_tickets.append(ticket_meta)
else:
if issue.fields.status.name in constants.JIRA_OPENED_STATUSES:
if not self._ticket_in_list(ticket_meta,
existing_tickets):
existing_tickets.append(ticket_meta)
except: # pylint: disable=W0702
log.exception(
f"Failed to create ticket for {finding['title']}")
error = Error(
tool=self.get_name(),
error=f"Failed to create ticket for {finding['title']}",
details=f"```\n{traceback.format_exc()}\n```")
self.errors.append(error)
self.set_meta("new_tickets", new_tickets)
self.set_meta("existing_tickets", existing_tickets)
self.set_meta("mapping", mapping_meta)
def create_issue(self,
title,
priority,
description,
issue_hash,
attachments=None,
get_or_create=True,
additional_labels=None):
def replace_defaults(value):
if isinstance(value,
str) and const.JIRA_FIELD_USE_DEFAULT_VALUE in value:
for default_key in default_fields.keys():
if default_key in value:
value = value.replace(default_key,
default_fields[default_key])
return value
default_fields = {
'!default_issuetype': 'Bug',
'!default_summary': title,
'!default_description': description,
'!default_priority': priority
}
description = self.markdown_to_jira_markdown(description)
issue_data = {
'project': {
'key': self.project
},
'issuetype': 'Bug',
'summary': title,
'description': description,
'priority': {
'name': priority
}
}
fields = deepcopy(self.fields)
for key, value in fields.items():
if isinstance(value, str):
if const.JIRA_FIELD_DO_NOT_USE_VALUE in value:
issue_data.pop(key)
else:
issue_data[key] = replace_defaults(value)
elif isinstance(value, list):
for item in value:
value[value.index(item)] = replace_defaults(item)
if issue_data.get(key):
issue_data[key].extend(value)
else:
issue_data[key] = value
elif isinstance(value, dict):
for _key, _value in value.items():
value[_key] = replace_defaults(_value)
issue_data[key] = value
elif not key in issue_data:
issue_data[key] = value
else:
logging.warning(
'field {} is already set and has \'{}\' value'.format(
key, issue_data[key]))
_labels = []
if additional_labels and isinstance(additional_labels, list):
_labels.extend(additional_labels)
if issue_data.get('labels', None):
issue_data['labels'].extend(_labels)
else:
issue_data['labels'] = _labels
jira_request = self.JIRA_REQUEST.format(issue_data["project"]["key"],
issue_hash, issue_hash)
if get_or_create:
issue, created = self.get_or_create_issue(jira_request, issue_data)
else:
issue = self.post_issue(issue_data)
created = True
try:
if attachments:
for attachment in attachments:
if 'binary_content' in attachment:
self.add_attachment(
issue.key,
attachment=attachment['binary_content'],
filename=attachment['message'])
for watcher in self.watchers:
self.client.add_watcher(issue.id, watcher)
except:
if os.environ.get("debug", False):
logging.error(format_exc())
finally:
try:
result_priority = issue.fields.priority
except:
result_priority = "Default"
self.created_jira_tickets.append({
'description':
issue.fields.summary,
'priority':
result_priority,
'key':
issue.key,
'link':
self.url + '/browse/' + issue.key,
'new':
created,
'assignee':
issue.fields.assignee,
'status':
issue.fields.status.name,
'open_date':
issue.fields.created
})
return issue, created
