def _active_scan(self): log.info("Active scan against target %s", self.config.get("target")) if self.config.get("auth_script", None): scan_id = self._zap_api.ascan.scan_as_user( self.config.get("target"), self._zap_context, self._zap_user, recurse=True, scanpolicyname=self._scan_policy_name) else: scan_id = self._zap_api.ascan.scan( self.config.get("target"), scanpolicyname=self._scan_policy_name) # try: int(scan_id) except: # pylint: disable=W0702 log.warning( "ZAP failed to return scan ID (scan_id=%s). Please check that target URL is accessible from Carrier DAST container", scan_id) # pylint: disable=C0301 return # status.wait_for_completion( lambda: int(self._zap_api.ascan.status(scan_id)) < 100, lambda: int(self._zap_api.ascan.status(scan_id)), "Active scan progress: %d%%")
def prepare_jira_mapping(jira_service): """ Make Jira mapping (for projects that are using custom values) """ if not jira_service or not jira_service.valid: return dict() jira_service.connect() issue_type = "Bug" if jira_service.fields["issuetype"]["name"] != "!default_issuetype": issue_type = jira_service.fields["issuetype"]["name"] project_priorities = get_project_priorities(jira_service.client, jira_service.project, issue_type) if not project_priorities: jira_service.client.close() return dict() logging.debug("%s %s priorities: %s", jira_service.project, issue_type, str(project_priorities)) c = const mapping = dict() for severity in c.JIRA_SEVERITIES: if severity not in project_priorities: for alternative in c.JIRA_ALTERNATIVES[severity]: if alternative in project_priorities: logging.warning("Mapping %s %s Jira priority: %s -> %s", jira_service.project, issue_type, severity, alternative) mapping[severity] = alternative break if severity not in mapping: mapping[severity] = project_priorities[0] logging.error( "Failed to find Jira mapping for %s, using %s as a fallback", severity, mapping[severity]) jira_service.client.close() return mapping
def execute(self): """ Run the scanner """ # Discover open ports include_ports = list() if self.config.get("include_ports", "0-65535"): include_ports.append( f'-p{self.config.get("include_ports", "0-65535")}') exclude_ports = list() if self.config.get("exclude_ports", None): exclude_ports.append("--exclude-ports") exclude_ports.append(f'{self.config.get("exclude_ports")}') target_url = url.parse_url(self.config.get("target")) task = subprocess.run( ["nmap", "-PN"] + include_ports + exclude_ports + [ "--min-rate", "1000", "--max-retries", "0", "--max-rtt-timeout", "200ms", target_url.hostname ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.log_subprocess_result(task) # Use discovered ports ports = list() tcp_ports = "" udp_ports = "" for each in re.findall(r'([0-9]*/[tcp|udp])', str(task.stdout)): if "/t" in each: tcp_ports += f'{each.replace("/t", "")},' elif "/u" in each: udp_ports += f'{each.replace("/u", "")},' if tcp_ports: ports.append(f"-pT:{tcp_ports[:-1]}") if udp_ports: ports.append(f"-pU:{udp_ports[:-1]}") if not ports: log.warning("No open ports found. Exiting") return # Make temporary files output_file_fd, output_file = tempfile.mkstemp() log.debug("Output file: %s", output_file) os.close(output_file_fd) # Scan target nmap_parameters = shlex.split( self.config.get("nmap_parameters", "-v -sVA")) nse_scripts = self.config.get( "nse_scripts", "ssl-date,http-mobileversion-checker,http-robots.txt,http-title,http-waf-detect," "http-chrono,http-headers,http-comments-displayer,http-date") task = subprocess.run(["nmap"] + nmap_parameters + ports + [ "--min-rate", "1000", "--max-retries", "0", f'--script={nse_scripts}', target_url.hostname, "-oX", output_file ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.log_subprocess_result(task) # Parse findings parse_findings(output_file, self) # Save intermediates self.save_intermediates(output_file, task) # Remove temporary files os.remove(output_file)
def parse_findings(data, scanner): """ Parse findings """ # Parse JSON using legacy parser try: findings = BrakemanParser(data).items except: # pylint: disable=W0702 log.exception("Failed to parse brakeman report") log.warning( "Possibly ruby code path is invalid or not Ruby-on-Rails application" ) findings = list() # Make finding instances for item in findings: finding = SastFinding( title=item["title"], description=[ "\n\n".join([ markdown.markdown_escape(item['description']), f"**References:** {markdown.markdown_escape(item['references'])}", f"**File to review:** {markdown.markdown_escape(item['file_path'])}" \ f":{item['line']}" ]) ] ) finding.set_meta("tool", scanner.get_name()) finding.set_meta("severity", constants.BRAKEMAN_SEVERITY_MAPPING[item["severity"]]) finding.set_meta("legacy.file", item["file_path"]) finding.set_meta("legacy.line", item["line"]) finding.set_meta( "endpoints", [namedtuple("Endpoint", ["raw"])(raw=item["file_path"])]) log.debug(f"Endpoints: {finding.get_meta('endpoints')}") scanner.findings.append(finding)
def execute(self, args): """ Run the command """ log.info("Starting") if args.call_from_legacy: log.warning("Called from legacy entry point") # Make instances context = RunContext(args) config = ConfigHelper(context) scanning = ScanningPerformer(context) processing = ProcessingPerformer(context) reporting = ReportingPerformer(context) # Add to context context.performers["scanning"] = scanning context.performers["processing"] = processing context.performers["reporting"] = reporting # Init config config.load(args.config_variable, args.config_file, args.suite) scanning.validate_config(context.config) processing.validate_config(context.config) reporting.validate_config(context.config) # Prepare scanning.prepare() processing.prepare() reporting.prepare() # Perform scanning.perform() processing.perform() reporting.perform() # Done log.info("Done")
def connect(self): """ Establish connection to SMTP server """ try: self.connection = smtplib.SMTP(self.server, self.port, timeout=self.timeout) self.connection.ehlo() self.connection.starttls(context=ssl.create_default_context()) self.connection.ehlo() self.connection.login(self.login, self.password) except ssl.SSLError: log.warning("SSL error, retrying with unverified SSL context") self.connection = smtplib.SMTP(self.server, self.port, timeout=self.timeout) self.connection.ehlo() self.connection.starttls(context=ssl._create_unverified_context()) # pylint: disable=W0212 self.connection.ehlo() self.connection.login(self.login, self.password) except smtplib.SMTPServerDisconnected: log.warning( "Seems like SMTP with TSL didn't work, trying with SMTP_SSL") self.connection = smtplib.SMTP_SSL(host=self.server, port=self.port, timeout=self.timeout) self.connection.ehlo() self.connection.login(self.login, self.password) except: # pylint: disable=W0702 log.exception("Failed to connect to SMTP server") error = Error(tool="EMail", error="Failed to connect to SMTP server", details=f"```\n{traceback.format_exc()}\n```") self.context.errors.append(error) if self.connection: self.connection.quit()
def report(self): """ Report """ if not self._rp_client: log.warning( "ReportPortal configuration/connection is invalid. Skipping RP reporting" ) return log.info("Reporting to ReportPortal") for item in self.context.findings: if item.get_meta("information_finding", False) or \ item.get_meta("false_positive_finding", False) or \ item.get_meta("excluded_finding", False): continue if isinstance(item, DastFinding): item_details = markdown.markdown_unescape(item.description) item_description = item_details tags = [ f'Tool: {item.get_meta("tool", "")}', f'TestType: {self.context.get_meta("testing_type", "DAST")}', f'Severity: {item.get_meta("severity", SEVERITIES[-1])}' ] if item.get_meta("confidence", None): tags.append(f'Confidence: {item.get_meta("confidence")}') self._rp_client.start_test_item(item.title, description=item_description, tags=tags) if item.get_meta("legacy.images", None): for attachment in item.get_meta("legacy.images"): self._rp_client.test_item_message( attachment["name"], "INFO", attachment) self._rp_client.test_item_message( "!!!MARKDOWN_MODE!!! %s " % item_details, "INFO") self._rp_client.test_item_message( item.get_meta("issue_hash", "<no_hash>"), "ERROR") self._rp_client.finish_test_item() elif isinstance(item, SastFinding): item_details = markdown.markdown_unescape("\n\n".join( item.description)) item_description = item_details tags = [ f'Tool: {item.get_meta("tool", "")}', f'TestType: {self.context.get_meta("testing_type", "SAST")}', f'Severity: {item.get_meta("severity", SEVERITIES[-1])}' ] if item.get_meta("confidence", None): tags.append(f'Confidence: {item.get_meta("confidence")}') self._rp_client.start_test_item(item.title, description=item_description, tags=tags) self._rp_client.test_item_message( "!!!MARKDOWN_MODE!!! %s " % item_details, "INFO") self._rp_client.test_item_message( item.get_meta("issue_hash", "<no_hash>"), "ERROR") self._rp_client.finish_test_item() else: log.warning("Unsupported finding type") continue # raise ValueError("Unsupported item type") self._rp_client.finish_test()
def __init__(self, url, user, password, project, fields=None): self.valid = True self.url = url self.password = password self.user = user try: self.connect() except: logging.error("Failed to connect to Jira") self.valid = False return self.projects = [project.key for project in self.client.projects()] self.project = project.upper() if self.project not in self.projects: # self.client.close() logging.warning("Requested project not found in Jira projects") # self.valid = False # return self.fields = {} self.watchers = [] if isinstance(fields, dict): if 'watchers' in fields.keys(): self.watchers = [item.strip() for item in fields.pop('watchers').split(",")] all_jira_fields = self.client.fields() for key, value in fields.items(): if value: if isinstance(value, str) and const.JIRA_FIELD_DO_NOT_USE_VALUE in value: self.fields[key] = value continue jira_keys = [item for item in all_jira_fields if item["id"] == key] if not jira_keys: jira_keys = [item for item in all_jira_fields if item["name"].lower() == key.lower().replace('_', ' ')] if len(jira_keys) == 1: jira_key = jira_keys[0] key_type = jira_key['schema']['type'] else: logging.warning(f'Cannot recognize field {key}. This field will not be used.') continue if key_type in ['string', 'number', 'any'] or isinstance(value, dict): _value = value elif key_type == 'array': if isinstance(value, str): _value = [item.strip() for item in value.split(",")] elif isinstance(value, int): _value = [value] else: _value = value else: _value = {'name': value} self.fields[jira_key['id']] = _value if not self.fields.get('issuetype', None): self.fields['issuetype'] = {'name': '!default_issuetype'} self.client.close() self.created_jira_tickets = list()
def execute(self, args): """ Run the command """ log.debug("Starting") if args.call_from_legacy: log.warning("Called from legacy entry point") # Init context context = RunContext(args) config = ConfigModel(context) if args.list_suites or not args.suite: suites = config.list_suites(args.config_seed, args.config_variable, args.config_file) if not args.suite: log.error("Suite is not defined. Use --help to get help") log.info("Available suites: %s", ", ".join(suites)) return # Make instances scanning = ScanningPerformer(context) processing = ProcessingPerformer(context) reporting = ReportingPerformer(context) # Add to context context.performers["scanning"] = scanning context.performers["processing"] = processing context.performers["reporting"] = reporting # Init config config.load(args.config_seed, args.config_variable, args.config_file, args.suite) scanning.validate_config(context.config) processing.validate_config(context.config) reporting.validate_config(context.config) # Add meta to context self._fill_context_meta(context) # Load state context.state.load() # Prepare reporters first reporting.prepare() # Run actions actions.run(context) # Prepare scanning and processing scanning.prepare() processing.prepare() # Perform scanning.perform() processing.perform() reporting.perform() # Done context.state.save() reporting.flush() log.debug("Done") # Show quality gate statistics if any for line in context.get_meta("quality_gate_stats", list()): log.info(line) # Fail quality gate if needed if context.get_meta("fail_quality_gate", False): os._exit(1) # pylint: disable=W0212
def execute(self): """ Run the processor """ log.info("Checking quality gate status") thresholds = self.config.get("thresholds", dict()) # Count issues by severity results_by_severity = dict() for item in self.context.findings: if item.get_meta("information_finding", False) or \ item.get_meta("false_positive_finding", False) or \ item.get_meta("excluded_finding", False): continue severity = item.get_meta("severity", SEVERITIES[-1]) if severity not in results_by_severity: results_by_severity[severity] = 0 results_by_severity[severity] += 1 # Prepare stats data stats_data = dict() for severity in SEVERITIES: stats_data["total"] = "OK" stats_data[severity] = { "findings": results_by_severity.get(severity, "-"), "threshold": thresholds.get(severity, "-"), "status": "OK" } # Check quality gate for severity in results_by_severity: if severity not in thresholds: continue # severity_results = results_by_severity[severity] policy_threshold = thresholds[severity] # if severity_results > policy_threshold: log.warning("Quality gate failed: %s -> %d > %d", severity, severity_results, policy_threshold) self.context.set_meta("fail_quality_gate", True) stats_data[severity]["status"] = "FAIL" stats_data["total"] = "FAIL" # Prepare stats stats = list() stats.append("============= Quality gate stats ============") stats.append( "Severity : {:<9} {:<5} {:<7} {:<4} {:<4}".format(*SEVERITIES)) stats.append("Findings : {:<9} {:<5} {:<7} {:<4} {:<4}".format( *[stats_data[severity]["findings"] for severity in SEVERITIES])) stats.append("Threshold : {:<9} {:<5} {:<7} {:<4} {:<4}".format( *[stats_data[severity]["threshold"] for severity in SEVERITIES])) stats.append("Status : {:<9} {:<5} {:<7} {:<4} {:<4}".format( *[stats_data[severity]["status"] for severity in SEVERITIES])) stats.append("============= Quality gate: {:<4} ============".format( stats_data["total"])) self.context.set_meta("quality_gate_stats", stats)
def execute(self): """ Run the scanner """ # Get target host IP target_url = url.parse_url(self.config.get("target")) host = target_url.hostname if not url.find_ip(host): task = subprocess.run(["getent", "hosts", host], stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.log_subprocess_result(task) host = url.find_ip(task.stdout.decode("utf-8", errors="ignore")) if host: host = host[0].strip() if not host: log.warning("No target IP found. Exiting") return # Prepare config include_ports = list() if self.config.get("include_ports", "0-65535"): include_ports.append("-p") include_ports.append( f'{self.config.get("include_ports", "0-65535")}') include_ports.append( f'-pU:{self.config.get("include_ports", "0-65535")}') exclude_ports = list() if self.config.get("exclude_ports", None): exclude_ports.append("--exclude-ports") exclude_ports.append(f'{self.config.get("exclude_ports")}') # Make temporary files output_file_fd, output_file = tempfile.mkstemp() log.debug("Output file: %s", output_file) os.close(output_file_fd) # Scan target task = subprocess.run(["masscan", host] + include_ports + [ "--rate", "1000", "-oJ", output_file, ] + exclude_ports, stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.log_subprocess_result(task) # Parse findings parse_findings(output_file, self) # Save intermediates self.save_intermediates(output_file, task) # Remove temporary files os.remove(output_file)
def _spider(self): log.info("Spidering target: %s", self.config.get("target")) if self.config.get("auth_script", None): scan_id = self._zap_api.spider.scan_as_user( self._zap_context, self._zap_user, self.config.get("target"), recurse=True, subtreeonly=True) else: scan_id = self._zap_api.spider.scan(self.config.get("target")) # try: int(scan_id) except: # pylint: disable=W0702 log.warning( "ZAP failed to return scan ID (scan_id=%s). Please check that target URL is accessible from Carrier DAST container", scan_id) # pylint: disable=C0301 return # status.wait_for_completion( lambda: int(self._zap_api.spider.status(scan_id)) < 100, lambda: int(self._zap_api.spider.status(scan_id)), "Spidering progress: %d%%")
def run(self): """ Run action """ # Patch dulwich to work without valid UID/GID dulwich.repo.__original__get_default_identity = dulwich.repo._get_default_identity # pylint: disable=W0212 dulwich.repo._get_default_identity = _dulwich_repo_get_default_identity # pylint: disable=W0212 # Patch dulwich to use paramiko SSH client dulwich.client.get_ssh_vendor = ParamikoSSHVendor # Patch paramiko to skip key verification paramiko.transport.Transport._verify_key = _paramiko_transport_verify_key # pylint: disable=W0212 # Set USERNAME if needed try: getpass.getuser() except: # pylint: disable=W0702 os.environ["USERNAME"] = "******" # Get options source = self.config.get("source") target = self.config.get("target") branch = self.config.get("branch", "master") depth = self.config.get("depth", None) # Prepare auth auth_args = dict() if self.config.get("username", None) is not None: auth_args["username"] = self.config.get("username") if self.config.get("password", None) is not None: auth_args["password"] = self.config.get("password") if self.config.get("key", None) is not None: auth_args["key_filename"] = self.config.get("key") if self.config.get("key_data", None) is not None: key_obj = io.StringIO( self.config.get("key_data").replace("|", "\n")) pkey = paramiko.RSAKey.from_private_key(key_obj) # Patch paramiko to use our key paramiko.client.SSHClient._auth = _paramiko_client_SSHClient_auth( # pylint: disable=W0212 paramiko.client.SSHClient._auth, pkey # pylint: disable=W0212 ) # Clone repository log.info("Cloning repository %s into %s", source, target) repository = porcelain.clone(source, target, checkout=False, depth=depth, errstream=log.DebugLogStream(), **auth_args) # Get current HEAD tree (default branch) try: head_tree = repository[b"HEAD"] except: # pylint: disable=W0702 head_tree = None # Get target tree (requested branch) branch_b = branch.encode("utf-8") try: target_tree = repository[b"refs/remotes/origin/" + branch_b] except: # pylint: disable=W0702 target_tree = None # Checkout branch if target_tree is not None: log.info("Checking out branch %s", branch) repository[b"refs/heads/" + branch_b] = repository[b"refs/remotes/origin/" + branch_b] repository.refs.set_symbolic_ref(b"HEAD", b"refs/heads/" + branch_b) repository.reset_index(repository[b"HEAD"].tree) elif head_tree is not None: try: default_branch_name = repository.refs.follow(b"HEAD")[0][1] if default_branch_name.startswith(refs.LOCAL_BRANCH_PREFIX): default_branch_name = default_branch_name[ len(refs.LOCAL_BRANCH_PREFIX):] default_branch_name = default_branch_name.decode("utf-8") log.warning( "Branch %s was not found. Checking out default branch %s", branch, default_branch_name) except: # pylint: disable=W0702 log.warning( "Branch %s was not found. Trying to check out default branch", branch) try: repository.reset_index(repository[b"HEAD"].tree) except: # pylint: disable=W0702 log.exception("Failed to checkout default branch") else: log.error( "Branch %s was not found and default branch is not set. Skipping checkout" ) # Delete .git if requested if self.config.get("delete_git_dir", False): log.info("Deleting .git directory") shutil.rmtree(os.path.join(target, ".git"))
def report_multi(self): """ Report """ wrappers_config = dict() wrappers_config[None] = self.config for key, value in self.config.get("dynamic_jira").items(): wrappers_config[re.compile(key)] = value # wrappers = dict() for wrapper_key, wrapper_config in wrappers_config.items(): wrapper = JiraWrapper( wrapper_config.get("url"), wrapper_config.get("username"), wrapper_config.get("password"), wrapper_config.get("project"), wrapper_config.get("fields"), ) # if not wrapper.valid: # Save default mapping to meta as a fallback default_mapping = constants.JIRA_SEVERITY_MAPPING default_mapping.update( self.config.get("custom_mapping", dict())) self.set_meta("mapping", default_mapping) # Report error log.error( "Jira configuration is invalid. Skipping Jira reporting") raise RuntimeError("Jira configuration is invalid") # wrappers[wrapper_key] = dict() wrappers[wrapper_key]["wrapper"] = wrapper wrappers[wrapper_key]["config"] = wrapper_config # if wrapper_config.get("separate_epic_linkage", False) and \ "Epic Link" in wrapper_config.get("fields"): wrappers[wrapper_key]["epic_link"] = wrapper_config.get( "fields").pop("Epic Link") # wrappers[wrapper_key]["raw_epic_link"] = None if wrapper_config.get("separate_epic_linkage", False): wrappers[wrapper_key]["raw_epic_link"] = wrappers[wrapper_key][ "epic_link"] elif "Epic Link" in wrapper_config.get("fields"): wrappers[wrapper_key]["raw_epic_link"] = wrapper_config.get( "fields")["Epic Link"] # wrappers[wrapper_key]["priority_mapping"] = wrapper_config.get( "custom_mapping", prepare_jira_mapping(wrapper)) wrappers[wrapper_key]["mapping_meta"] = dict( wrappers[wrapper_key]["priority_mapping"]) # self.set_meta("wrapper", wrappers[None]["wrapper"]) self.set_meta("raw_epic_link", wrappers[None]["raw_epic_link"]) # dynamic_label_mapping = dict() if self.config.get("dynamic_labels", None): try: for key, value in self.config.get("dynamic_labels").items(): dynamic_label_mapping[re.compile(key)] = value except: # pylint: disable=W0702 log.exception("Failed to add dynamic label mapping") # dynamic_field_mapping = dict() if self.config.get("dynamic_fields", None): try: for key, value in self.config.get("dynamic_fields").items(): dynamic_field_mapping[re.compile(key)] = value except: # pylint: disable=W0702 log.exception("Failed to add dynamic field mapping") # findings = list() for item in self.context.findings: # pylint: disable=R1702 # if item.get_meta("information_finding", False) or \ item.get_meta("false_positive_finding", False) or \ item.get_meta("excluded_finding", False): continue # if isinstance(item, (DastFinding, SastFinding)): # dynamic_labels = list() dynamic_fields = list() dynamic_wrapper = wrappers[None] # for endpoint in item.get_meta("endpoints", list()): # for pattern, addon_label in dynamic_label_mapping.items(): try: if pattern.match(endpoint.raw): dynamic_labels.append(addon_label) except: # pylint: disable=W0702 log.exception("Failed to add dynamic label") # for pattern, addon_fields in dynamic_field_mapping.items(): try: if pattern.match(endpoint.raw): dynamic_fields.append(addon_fields) except: # pylint: disable=W0702 log.exception("Failed to add dynamic field") # for pattern, addon_jira in wrappers.items(): if pattern is None: continue try: if pattern.match(endpoint.raw): dynamic_wrapper = addon_jira except: # pylint: disable=W0702 log.exception("Failed to add dynamic JIRA") # severity = item.get_meta("severity", SEVERITIES[-1]) priority = constants.JIRA_SEVERITY_MAPPING[severity] if dynamic_wrapper["priority_mapping"] and \ priority in dynamic_wrapper["priority_mapping"]: priority = dynamic_wrapper["priority_mapping"][priority] dynamic_wrapper["mapping_meta"][severity] = priority # if isinstance(item, DastFinding): findings.append({ "title": item.title, "priority": priority, "description": item.description.replace("\\.", "."), "issue_hash": item.get_meta("issue_hash", "<no_hash>"), "additional_labels": [ label.replace(" ", "_") for label in [ item.get_meta("tool", "scanner"), self.context.get_meta("testing_type", "DAST"), item.get_meta("severity", SEVERITIES[-1]) ] ] + dynamic_labels, "dynamic_fields": dynamic_fields, "raw": item, "wrapper": dynamic_wrapper, }) elif isinstance(item, SastFinding): # description_chunks = [ item.replace("\\.", ".").replace( "<pre>", "{code:collapse=true}\n\n").replace( "</pre>", "\n\n{code}").replace("<br />", "\n") for item in item.description ] # if len("\n\n".join(description_chunks) ) > constants.JIRA_DESCRIPTION_MAX_SIZE: description = description_chunks[0] chunks = description_chunks[1:] comments = list() new_line_str = ' \n \n' for chunk in chunks: if not comments or ( len(comments[-1]) + len(new_line_str) + len(chunk) ) >= \ constants.JIRA_COMMENT_MAX_SIZE: comments.append(cut_jira_comment(chunk)) else: # Last comment can handle one more chunk comments[ -1] += new_line_str + cut_jira_comment( chunk) else: description = "\n\n".join(description_chunks) comments = list() # findings.append({ "title": item.title, "priority": priority, "description": description, "issue_hash": item.get_meta("issue_hash", "<no_hash>"), "additional_labels": [ label.replace(" ", "_") for label in [ item.get_meta("tool", "scanner"), self.context.get_meta("testing_type", "SAST"), item.get_meta("severity", SEVERITIES[-1]) ] ] + dynamic_labels, "dynamic_fields": dynamic_fields, "comments": comments, "raw": item, "wrapper": dynamic_wrapper, }) # # else: log.warning("Unsupported finding type") continue # raise ValueError("Unsupported item type") # self.set_meta("mapping", wrappers[None]["mapping_meta"]) # for finding in findings: if finding["wrapper"]["config"].get("max_description_size", False): if len(finding["description"]) > \ int(finding["wrapper"]["config"].get("max_description_size")): if "comments" not in finding: finding["comments"] = list() # comment_chunks = list() cut_line_len = len(constants.JIRA_DESCRIPTION_CUT) cut_point = int(finding["wrapper"]["config"].get( "max_description_size")) - cut_line_len # item_description = finding["description"] finding["description"] = \ f"{item_description[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}" # description_data = item_description[cut_point:] comment_cut_threshold = min( constants.JIRA_COMMENT_MAX_SIZE, int(finding["wrapper"]["config"].get( "max_description_size"))) cut_point = comment_cut_threshold - cut_line_len # while description_data: if len(description_data) > comment_cut_threshold: comment_chunks.append( f"{description_data[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}" ) description_data = description_data[cut_point:] else: comment_chunks.append(description_data) break # while comment_chunks: finding["comments"].insert(0, comment_chunks.pop()) # findings.sort(key=lambda item: (SEVERITIES.index(item["raw"].get_meta( "severity", SEVERITIES[-1])), item["raw"].get_meta("tool", ""), item["raw"].title)) # new_tickets = list() existing_tickets = list() # for _, local_wrapper in wrappers.items(): local_wrapper["wrapper"].connect() # for finding in findings: try: config_labels = finding["wrapper"]["config"].get( "additional_labels", None) if config_labels is None: config_labels = list() if not isinstance(config_labels, list): config_labels = [ item.strip() for item in config_labels.split(",") ] # field_overrides = dict() for dynamic_field in finding["dynamic_fields"]: field_overrides.update(dynamic_field) # issue, created = finding["wrapper"]["wrapper"].create_issue( finding["title"], # title finding["priority"], # priority finding["description"], # description finding["issue_hash"], # issue_hash, self.get_hash_code() # attachments=None, # get_or_create=True, additional_labels=finding["additional_labels"] + config_labels, # additional_labels # pylint: disable=C0301 field_overrides=field_overrides, ) if created and "comments" in finding: for comment in finding["comments"]: finding["wrapper"]["wrapper"].add_comment_to_issue( issue, comment) if created and finding["wrapper"]["config"].get( "separate_epic_linkage", False): try: finding["wrapper"][ "wrapper"].client.add_issues_to_epic( finding["wrapper"]["epic_link"], [str(issue.key)]) except: # pylint: disable=W0702 log.exception("Failed to add ticket %s to epic %s", str(issue.key), finding["wrapper"]["epic_link"]) try: result_priority = str(issue.fields.priority) except: # pylint: disable=W0702 result_priority = "Default" # ticket_meta = { "jira_id": issue.key, "jira_url": f"{finding['wrapper']['config'].get('url')}/browse/{issue.key}", # pylint: disable=C0301 "priority": result_priority, "status": issue.fields.status.name, "created": issue.fields.created, "open_date": datetime.strptime( issue.fields.created, "%Y-%m-%dT%H:%M:%S.%f%z").strftime("%d %b %Y %H:%M"), "description": issue.fields.summary, "assignee": str(issue.fields.assignee), "raw_created": str(issue.fields.created), "raw_severity": finding["raw"].get_meta("severity", SEVERITIES[-1]), "raw_jira_url": finding["wrapper"]["config"].get("url"), "raw_jira_project": finding["wrapper"]["config"].get("project"), "raw_jira_epic": finding["wrapper"]["raw_epic_link"], "raw_jira_fields": finding["wrapper"]["config"].get("fields"), "raw_addon_fields": field_overrides, "raw_addon_labels": finding["additional_labels"] + config_labels, } if created: if not self._ticket_in_list(ticket_meta, new_tickets): new_tickets.append(ticket_meta) else: if issue.fields.status.name in constants.JIRA_OPENED_STATUSES: if not self._ticket_in_list(ticket_meta, existing_tickets): existing_tickets.append(ticket_meta) except: # pylint: disable=W0702 log.exception( f"Failed to create ticket for {finding['title']}") error = Error( tool=self.get_name(), error=f"Failed to create ticket for {finding['title']}", details=f"```\n{traceback.format_exc()}\n```") self.errors.append(error) # self.set_meta("new_tickets", new_tickets) self.set_meta("existing_tickets", existing_tickets)
def validate_config(config): """ Validate config """ if "processing" not in config: log.warning("No processing defined in config")
def _process_depots(self, current_context_config): # pylint: disable=R0912 context_config = current_context_config # Check depot config section sanity if "depots" not in context_config["settings"]: context_config["settings"]["depots"] = dict() depot_sections = ["secret", "object", "state"] for depot_section in depot_sections: if depot_section not in context_config["settings"]["depots"]: context_config["settings"]["depots"][depot_section] = dict() # Support legacy depot configuration legacy_depot_names = [ item for item in list(context_config["settings"].get("depots", dict())) if item not in depot_sections ] legacy_section_map = { SecretDepotModel: "secret", ObjectDepotModel: "object", StateDepotModel: "state" } for depot_name in legacy_depot_names: try: depot_class = importlib.import_module( f"dusty.tools.depots.{depot_name}.depot").Depot for depot_type in legacy_section_map: if issubclass(depot_class, depot_type): depot_section = legacy_section_map[depot_type] context_config["settings"]["depots"][depot_section][depot_name] = \ context_config["settings"]["depots"][depot_name] log.info("Legacy depot %s added to section %s", depot_name, depot_section) context_config["settings"]["depots"].pop(depot_name) except: log.exception("Failed to process legacy depot %s", depot_name) # Make depot instances for depot_section in depot_sections: for depot_name in list( context_config["settings"]["depots"][depot_section]): try: depot_class = importlib.import_module( f"dusty.tools.depots.{depot_name}.depot").Depot if depot_class.get_name( ) in self.context.depots[depot_section]: continue # Depot already enabled in first iteration depot = depot_class( self.context, context_config["settings"]["depots"] [depot_section][depot_name]) self.context.depots[depot_section][ depot.get_name()] = depot log.info("Enabled %s depot %s", depot_section, depot_name) if isinstance(depot, SecretDepotModel): self.context.set_meta("depots_resolved_secrets", 0) context_config = self._depot_substitution( context_config) log.info( "Resolved %d secrets from depots", self.context.get_meta("depots_resolved_secrets", 0)) except ModuleNotFoundError: log.warning( "Depot %s is not enabled, probably needed module is not loaded yet", depot_name) except: log.exception("Failed to enable depot %s", depot_name) # return context_config
def report(self): """ Report """ # Remove "Epic Link" from fields if requested if self.config.get("separate_epic_linkage", False) and \ "Epic Link" in self.config.get("fields"): epic_link = self.config.get("fields").pop("Epic Link") # Prepare wrapper log.info("Creating legacy wrapper instance") wrapper = JiraWrapper(self.config.get("url"), self.config.get("username"), self.config.get("password"), self.config.get("project"), self.config.get("fields")) if not wrapper.valid: # Save default mapping to meta as a fallback default_mapping = constants.JIRA_SEVERITY_MAPPING default_mapping.update(self.config.get("custom_mapping", dict())) self.set_meta("mapping", default_mapping) # Report error log.error("Jira configuration is invalid. Skipping Jira reporting") raise RuntimeError("Jira configuration is invalid") log.debug("Legacy wrapper is valid") # Prepare findings priority_mapping = self.config.get("custom_mapping", prepare_jira_mapping(wrapper)) mapping_meta = dict(priority_mapping) findings = list() for item in self.context.findings: if item.get_meta("information_finding", False) or \ item.get_meta("false_positive_finding", False) or \ item.get_meta("excluded_finding", False): continue if isinstance(item, DastFinding): severity = item.get_meta("severity", SEVERITIES[-1]) priority = constants.JIRA_SEVERITY_MAPPING[severity] if priority_mapping and priority in priority_mapping: priority = priority_mapping[priority] mapping_meta[ severity] = priority # Update meta mapping to reflect actual results findings.append({ "title": item.title, "priority": priority, "description": item.description.replace("\\.", "."), "issue_hash": item.get_meta("issue_hash", "<no_hash>"), "additional_labels": [ label.replace(" ", "_") for label in [ item.get_meta("tool", "scanner"), self.context.get_meta("testing_type", "DAST"), item.get_meta("severity", SEVERITIES[-1]) ] ], "raw": item }) elif isinstance(item, SastFinding): severity = item.get_meta("severity", SEVERITIES[-1]) priority = constants.JIRA_SEVERITY_MAPPING[severity] if priority_mapping and priority in priority_mapping: priority = priority_mapping[priority] mapping_meta[ severity] = priority # Update meta mapping to reflect actual results description_chunks = [ item.replace("\\.", ".").replace( "<pre>", "{code:collapse=true}\n\n").replace( "</pre>", "\n\n{code}").replace("<br />", "\n") for item in item.description ] if len("\n\n".join(description_chunks) ) > constants.JIRA_DESCRIPTION_MAX_SIZE: description = description_chunks[0] chunks = description_chunks[1:] comments = list() new_line_str = ' \n \n' for chunk in chunks: if not comments or (len(comments[-1]) + len(new_line_str) + len(chunk)) >= \ constants.JIRA_COMMENT_MAX_SIZE: comments.append(cut_jira_comment(chunk)) else: # Last comment can handle one more chunk comments[-1] += new_line_str + cut_jira_comment( chunk) else: description = "\n\n".join(description_chunks) comments = list() findings.append({ "title": item.title, "priority": priority, "description": description, "issue_hash": item.get_meta("issue_hash", "<no_hash>"), "additional_labels": [ label.replace(" ", "_") for label in [ item.get_meta("tool", "scanner"), self.context.get_meta("testing_type", "SAST"), item.get_meta("severity", SEVERITIES[-1]) ] ], "comments": comments, "raw": item }) else: log.warning("Unsupported finding type") continue # raise ValueError("Unsupported item type") # Cut description if length above configured limit if self.config.get("max_description_size", False): for finding in findings: if len(finding["description"]) > int( self.config.get("max_description_size")): if "comments" not in finding: finding["comments"] = list() # comment_chunks = list() cut_line_len = len(constants.JIRA_DESCRIPTION_CUT) cut_point = int( self.config.get("max_description_size")) - cut_line_len # item_description = finding["description"] finding["description"] = \ f"{item_description[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}" # description_data = item_description[cut_point:] comment_cut_threshold = min( constants.JIRA_COMMENT_MAX_SIZE, int(self.config.get("max_description_size"))) cut_point = comment_cut_threshold - cut_line_len # while description_data: if len(description_data) > comment_cut_threshold: comment_chunks.append( f"{description_data[:cut_point]}{constants.JIRA_DESCRIPTION_CUT}" ) description_data = description_data[cut_point:] else: comment_chunks.append(description_data) break # while comment_chunks: finding["comments"].insert(0, comment_chunks.pop()) # Sort findings by severity-tool-title findings.sort(key=lambda item: (SEVERITIES.index(item["raw"].get_meta( "severity", SEVERITIES[-1])), item["raw"].get_meta("tool", ""), item["raw"].title)) # Submit issues wrapper.connect() new_tickets = list() existing_tickets = list() for finding in findings: try: issue, created = wrapper.create_issue( finding["title"], # title finding["priority"], # priority finding["description"], # description finding["issue_hash"], # issue_hash, self.get_hash_code() # attachments=None, # get_or_create=True, additional_labels=finding[ "additional_labels"] # additional_labels ) if created and "comments" in finding: for comment in finding["comments"]: wrapper.add_comment_to_issue(issue, comment) if created and self.config.get("separate_epic_linkage", False): try: wrapper.client.add_issues_to_epic( epic_link, [str(issue.key)]) except: # pylint: disable=W0702 log.exception("Failed to add ticket %s to epic %s", str(issue.key), epic_link) try: result_priority = issue.fields.priority except: # pylint: disable=W0702 result_priority = "Default" ticket_meta = { "jira_id": issue.key, "jira_url": f"{self.config.get('url')}/browse/{issue.key}", "priority": result_priority, "status": issue.fields.status.name, "created": issue.fields.created, "open_date": datetime.strptime( issue.fields.created, "%Y-%m-%dT%H:%M:%S.%f%z").strftime("%d %b %Y %H:%M"), "description": issue.fields.summary, "assignee": issue.fields.assignee } if created: if not self._ticket_in_list(ticket_meta, new_tickets): new_tickets.append(ticket_meta) else: if issue.fields.status.name in constants.JIRA_OPENED_STATUSES: if not self._ticket_in_list(ticket_meta, existing_tickets): existing_tickets.append(ticket_meta) except: # pylint: disable=W0702 log.exception( f"Failed to create ticket for {finding['title']}") error = Error( tool=self.get_name(), error=f"Failed to create ticket for {finding['title']}", details=f"```\n{traceback.format_exc()}\n```") self.errors.append(error) self.set_meta("new_tickets", new_tickets) self.set_meta("existing_tickets", existing_tickets) self.set_meta("mapping", mapping_meta)
def validate_config(config): """ Validate config """ if "scanners" not in config: log.warning("No scanners defined in config") config["scanners"] = dict()
def create_issue(self, title, priority, description, issue_hash, attachments=None, get_or_create=True, additional_labels=None): def replace_defaults(value): if isinstance(value, str) and const.JIRA_FIELD_USE_DEFAULT_VALUE in value: for default_key in default_fields.keys(): if default_key in value: value = value.replace(default_key, default_fields[default_key]) return value default_fields = { '!default_issuetype': 'Bug', '!default_summary': title, '!default_description': description, '!default_priority': priority } description = self.markdown_to_jira_markdown(description) issue_data = { 'project': { 'key': self.project }, 'issuetype': 'Bug', 'summary': title, 'description': description, 'priority': { 'name': priority } } fields = deepcopy(self.fields) for key, value in fields.items(): if isinstance(value, str): if const.JIRA_FIELD_DO_NOT_USE_VALUE in value: issue_data.pop(key) else: issue_data[key] = replace_defaults(value) elif isinstance(value, list): for item in value: value[value.index(item)] = replace_defaults(item) if issue_data.get(key): issue_data[key].extend(value) else: issue_data[key] = value elif isinstance(value, dict): for _key, _value in value.items(): value[_key] = replace_defaults(_value) issue_data[key] = value elif not key in issue_data: issue_data[key] = value else: logging.warning( 'field {} is already set and has \'{}\' value'.format( key, issue_data[key])) _labels = [] if additional_labels and isinstance(additional_labels, list): _labels.extend(additional_labels) if issue_data.get('labels', None): issue_data['labels'].extend(_labels) else: issue_data['labels'] = _labels jira_request = self.JIRA_REQUEST.format(issue_data["project"]["key"], issue_hash, issue_hash) if get_or_create: issue, created = self.get_or_create_issue(jira_request, issue_data) else: issue = self.post_issue(issue_data) created = True try: if attachments: for attachment in attachments: if 'binary_content' in attachment: self.add_attachment( issue.key, attachment=attachment['binary_content'], filename=attachment['message']) for watcher in self.watchers: self.client.add_watcher(issue.id, watcher) except: if os.environ.get("debug", False): logging.error(format_exc()) finally: try: result_priority = issue.fields.priority except: result_priority = "Default" self.created_jira_tickets.append({ 'description': issue.fields.summary, 'priority': result_priority, 'key': issue.key, 'link': self.url + '/browse/' + issue.key, 'new': created, 'assignee': issue.fields.assignee, 'status': issue.fields.status.name, 'open_date': issue.fields.created }) return issue, created
def execute(self): # pylint: disable=R0912 """ Run the scanner """ helper = QualysHelper( self.context, self.config.get("qualys_api_server"), self.config.get("qualys_login"), self.config.get("qualys_password"), retries=self.config.get("retries", 10), retry_delay=self.config.get("retry_delay", 30.0), timeout=self.config.get("timeout", 120) ) log.info("Qualys WAS version: %s", helper.get_version()) timestamp = datetime.utcfromtimestamp(int(time())).strftime("%Y-%m-%d %H:%M:%S") sleep_interval = self.config.get("sleep_interval", 10.0) status_check_interval = self.config.get("status_check_interval", 60.0) # Create/get project project_name = "{}_{}".format( self.context.get_meta("project_name", "UnnamedProject"), self.context.get_meta("project_description", "Undescribed Project") ) if self.config.get("random_name", False): project_name = f"{project_name}_{self.id_generator(8)}" log.info("Searching for existing webapp") webapp_id = helper.search_for_webapp(project_name) if webapp_id is None: log.info("Creating webapp") webapp_id = helper.create_webapp( project_name, self.config.get("target"), self.config.get("qualys_option_profile_id"), excludes=self.config.get("exclude", None) ) sleep(sleep_interval) # Create auth record if needed auth_id = None if self.config.get("auth_script", None): log.info("Creating auth record") auth_name = f"{project_name} SeleniumAuthScript {timestamp}" auth_data = self.render_selenium_script( self.config.get("auth_script"), self.config.get("auth_login", ""), self.config.get("auth_password", ""), self.config.get("target") ) auth_id = helper.create_selenium_auth_record( auth_name, auth_data, self.config.get("logged_in_indicator", "selenium") ) sleep(sleep_interval) helper.add_auth_record_to_webapp(webapp_id, project_name, auth_id) # Start scan log.info("Starting scan") scan_name = f"{project_name} WAS {timestamp}" scan_auth = {"isDefault": True} if auth_id is not None: scan_auth = {"id": auth_id} scan_scanner = {"type": "EXTERNAL"} if self.config.get("qualys_scanner_type", "EXTERNAL") == "INTERNAL" and \ self.config.get("qualys_scanner_pool", None): scanner_pool = self.config.get("qualys_scanner_pool") if isinstance(scanner_pool, str): scanner_pool = [item.strip() for item in scanner_pool.split(",")] scan_scanner = { "type": "INTERNAL", "friendlyName": random.choice(scanner_pool) } scan_id = helper.start_scan( scan_name, webapp_id, self.config.get("qualys_option_profile_id"), scan_scanner, scan_auth ) sleep(sleep_interval) # Wait for scan to finish while helper.get_scan_status(scan_id) in ["SUBMITTED", "RUNNING"]: log.info("Waiting for scan to finish") sleep(status_check_interval) # Wait for results to finish processing if helper.get_scan_results_status(scan_id) == "UNKNOWN": log.warning( "Unable to find scan results status. Scan status: %s", helper.get_scan_status(scan_id) ) while helper.get_scan_results_status(scan_id) in ["TO_BE_PROCESSED", "PROCESSING"]: log.info("Waiting for scan results to finish processing") sleep(status_check_interval) scan_result = helper.get_scan_results_status(scan_id) if scan_result in ["NO_HOST_ALIVE", "NO_WEB_SERVICE"]: error = Error( tool=self.get_name(), error=f"Qualys failed to access target", details="Qualys failed to access target " \ "(e.g. connection failed or target is not accessible). " \ "Please check scanner type/pool and target URL." ) self.errors.append(error) if scan_result in ["SCAN_RESULTS_INVALID", "SERVICE_ERROR", "SCAN_INTERNAL_ERROR"]: error = Error( tool=self.get_name(), error=f"Qualys internal error occured", details="Qualys failed to perform scan (internal scan error occured). " \ "Please re-run the scan and check config if error persists." ) self.errors.append(error) # Request report log.info("Requesting report") report_name = f"{project_name} WAS {timestamp} FOR Scan {scan_id}" report_id = helper.create_report( report_name, webapp_id, self.config.get("qualys_report_template_id") ) sleep(sleep_interval) # Wait for report to be created while helper.get_report_status(report_id) in ["RUNNING"]: log.info("Waiting for report to be created") sleep(status_check_interval) # Download report log.info("Downloading report XML") report_xml = helper.download_report(report_id) # Delete assets log.info("Deleting assets") helper.delete_asset("report", report_id) helper.delete_asset("wasscan", scan_id) if auth_id is not None: helper.delete_asset("webappauthrecord", auth_id) helper.delete_asset("webapp", webapp_id) # Parse findings parse_findings(report_xml, self) # Save intermediates self.save_intermediates(report_xml)
def validate_config(config): """ Validate config """ if "reporters" not in config: log.warning("No reporters defined in config") config["reporters"] = dict()