def prepare(self):
""" Prepare for action """
log.debug("Preparing")
config = self.context.config.get("processing")
config_items = [
item for item in list(config) if not isinstance(config[item], bool) or config[item]
]
disabled_items = [
item for item in list(config) if isinstance(config[item], bool) and not config[item]
]
# Schedule processors
try:
all_processors = dependency.resolve_name_order(
config_items + [
item for item in constants.DEFAULT_PROCESSORS if item not in disabled_items
], "dusty.processors.{}.processor", "Processor"
)
except:
all_processors = [
item for item in constants.DEFAULT_PROCESSORS if item not in disabled_items
] + config_items
for processor_name in all_processors:
try:
self.schedule_processor(processor_name, dict())
except:
log.exception("Failed to prepare processor %s", processor_name)
error = Error(
tool=processor_name,
error=f"Failed to prepare processor {processor_name}",
details=f"```\n{traceback.format_exc()}\n```"
)
self.context.errors.append(error)
# Resolve depencies once again
dependency.resolve_depencies(self.context.processors)
def parse_findings(data, scanner):
""" Parse findings """
# Get deps
deps = get_dependencies(
scanner.config.get("code"), scanner.config.get("add_devdep", False)
)
# Parse JSON using legacy parser
findings = NpmScanParser(data, deps).items
# Make finding instances
for item in findings:
finding = SastFinding(
title=item["title"],
description=[
"\n\n".join([
item['description'],
f"**URL:** {item['url']}",
f"**CWE:** {markdown.markdown_escape(item['cwe'])}",
f"**References:** {item['references']}",
f"**File to review:** {item['file_path']}"
])
]
)
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", constants.NPM_SEVERITY_MAPPING[item["severity"]])
finding.set_meta("legacy.file", item["file_path"])
endpoints = list()
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def parse_findings(data, scanner):
""" Parse findings """
log.debug("Parsing findings")
item_regex = re.compile(
"".join([
"^(\[\+\] New Finding!!!)$", "\s*Name: (?P<name>.*)$",
"\s*Url: (?P<url>.*)$",
"\s*Description: (?P<description>[\s\S]*?)\n\n"
]), re.MULTILINE)
for item in item_regex.finditer(data):
# Make finding object
description = list()
description.append(markdown.markdown_escape(item.group("description")))
description.append(
f'\n**URL:** {markdown.markdown_escape(item.group("url"))}')
description = "\n".join(description)
finding = DastFinding(title=item.group("name"),
description=description)
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", "Info")
# Endpoints (for backwards compatibility)
endpoints = list()
endpoint = url.parse_url(item.group("url"))
endpoints.append(endpoint)
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
# Done
scanner.findings.append(finding)
def schedule_reporter(self, reporter_name, reporter_config):
""" Schedule reporter run in current context after all already configured reporters """
try:
# Init reporter instance
reporter = importlib.import_module(
f"dusty.reporters.{reporter_name}.reporter").Reporter
if reporter.get_name() in self.context.reporters:
log.debug("Reporter %s already scheduled", reporter_name)
return
# Prepare config
config = self.context.config["reporters"]
if reporter_name not in config or not isinstance(
config[reporter_name], dict):
config[reporter_name] = dict()
if "reporters" in self.context.config["settings"]:
general_config = self.context.config["settings"]["reporters"]
merged_config = general_config.copy()
merged_config.update(config[reporter_name])
config[reporter_name] = merged_config
config[reporter_name].update(reporter_config)
# Validate config
reporter.validate_config(config[reporter_name])
# Add to context
self.context.reporters[reporter.get_name()] = reporter(
self.context)
# Resolve depencies
dependency.resolve_depencies(self.context.reporters)
# Done
log.debug("Scheduled reporter %s", reporter_name)
except:
log.exception("Failed to schedule reporter %s", reporter_name)
error = Error(tool=reporter_name,
error=f"Failed to schedule reporter {reporter_name}",
details=f"```\n{traceback.format_exc()}\n```")
self.context.errors.append(error)
def _start_zap(self):
""" Start ZAP daemon, create API client """
log.info("Starting ZAP daemon")
bind_host = "127.0.0.1"
if self.config.get("bind_all_interfaces", True):
bind_host = "0.0.0.0"
daemon_out = subprocess.DEVNULL
if self.config.get("daemon_debug", False):
daemon_out = sys.stdout
zap_home_dir = tempfile.mkdtemp()
log.debug("ZAP home directory: %s", zap_home_dir)
self._zap_daemon = subprocess.Popen([
"/usr/bin/java", self.config.get("java_options", "-Xmx499m"),
"-jar", constants.ZAP_PATH,
"-dir", zap_home_dir,
"-daemon", "-port", "8091", "-host", bind_host,
"-config", "api.key=dusty",
"-config", "api.addrs.addr.regex=true",
"-config", "api.addrs.addr.name=.*",
"-config", "ajaxSpider.browserId=htmlunit"
], stdout=daemon_out, stderr=daemon_out)
self._zap_api = ZAPv2(
apikey="dusty",
proxies={
"http": "http://127.0.0.1:8091",
"https": "http://127.0.0.1:8091"
}
)
def execute(self):
""" Run the scanner """
# Discover open ports
include_ports = list()
if self.config.get("include_ports", "0-65535"):
include_ports.append(
f'-p{self.config.get("include_ports", "0-65535")}')
exclude_ports = list()
if self.config.get("exclude_ports", None):
exclude_ports.append("--exclude-ports")
exclude_ports.append(f'{self.config.get("exclude_ports")}')
target_url = url.parse_url(self.config.get("target"))
task = subprocess.run(
["nmap", "-PN"] + include_ports + exclude_ports + [
"--min-rate", "1000", "--max-retries", "0",
"--max-rtt-timeout", "200ms", target_url.hostname
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.log_subprocess_result(task)
# Use discovered ports
ports = list()
tcp_ports = ""
udp_ports = ""
for each in re.findall(r'([0-9]*/[tcp|udp])', str(task.stdout)):
if "/t" in each:
tcp_ports += f'{each.replace("/t", "")},'
elif "/u" in each:
udp_ports += f'{each.replace("/u", "")},'
if tcp_ports:
ports.append(f"-pT:{tcp_ports[:-1]}")
if udp_ports:
ports.append(f"-pU:{udp_ports[:-1]}")
if not ports:
log.warning("No open ports found. Exiting")
return
# Make temporary files
output_file_fd, output_file = tempfile.mkstemp()
log.debug("Output file: %s", output_file)
os.close(output_file_fd)
# Scan target
nmap_parameters = shlex.split(
self.config.get("nmap_parameters", "-v -sVA"))
nse_scripts = self.config.get(
"nse_scripts",
"ssl-date,http-mobileversion-checker,http-robots.txt,http-title,http-waf-detect,"
"http-chrono,http-headers,http-comments-displayer,http-date")
task = subprocess.run(["nmap"] + nmap_parameters + ports + [
"--min-rate", "1000", "--max-retries", "0",
f'--script={nse_scripts}', target_url.hostname, "-oX", output_file
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.log_subprocess_result(task)
# Parse findings
parse_findings(output_file, self)
# Save intermediates
self.save_intermediates(output_file, task)
# Remove temporary files
os.remove(output_file)
def parse_findings(data, scanner):
""" Parse findings """
# Parse JSON using legacy parser
try:
findings = BrakemanParser(data).items
except: # pylint: disable=W0702
log.exception("Failed to parse brakeman report")
log.warning(
"Possibly ruby code path is invalid or not Ruby-on-Rails application"
)
findings = list()
# Make finding instances
for item in findings:
finding = SastFinding(
title=item["title"],
description=[
"\n\n".join([
markdown.markdown_escape(item['description']),
f"**References:** {markdown.markdown_escape(item['references'])}",
f"**File to review:** {markdown.markdown_escape(item['file_path'])}" \
f":{item['line']}"
])
]
)
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity",
constants.BRAKEMAN_SEVERITY_MAPPING[item["severity"]])
finding.set_meta("legacy.file", item["file_path"])
finding.set_meta("legacy.line", item["line"])
finding.set_meta(
"endpoints",
[namedtuple("Endpoint", ["raw"])(raw=item["file_path"])])
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def _depot_read_config_object(self, obj):
result = dict()
if obj is None:
return result
data = depots.get_object(self.context, obj)
if data is None:
return result
try:
self.context.set_meta("depots_resolved_secrets", 0)
result = self._depot_substitution(
self._variable_substitution(
yaml.load(
os.path.expandvars(data),
Loader=yaml.FullLoader
)
)
)
log.info("Loaded %s from depots", obj)
log.debug(
"Resolved %d object secrets from depots",
self.context.get_meta("depots_resolved_secrets", 0)
)
return result
except:
return result
def parse_findings(result, scanner):
""" Parse findings """
# Parse JSON using legacy parser
findings = NodeJsScanParser(result).items
# Make finding instances
for item in findings:
finding = SastFinding(
title=item["title"],
description=[
"\n\n".join([
markdown.markdown_escape(item['description']),
f"**File to review:** {markdown.markdown_escape(item['file_path'])}" \
f":{item['line']}"
])
] + item["steps_to_reproduce"]
)
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", item["severity"])
finding.set_meta("legacy.file", item["file_path"])
finding.set_meta("legacy.line", item["line"])
endpoints = list()
if item["file_path"]:
endpoints.append(
namedtuple("Endpoint", ["raw"])(raw=item["file_path"]))
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def parse_findings(data, scanner):
""" Parse findings """
try:
findings = GitleaksScanParser(data, scanner).items
# Make finding instances
for item in findings:
description = item["description"]
if scanner.config.get("additional_text", None):
description = scanner.config.get(
"additional_text") + "\n\n" + description
finding = SastFinding(
title=item["title"],
description=[
"\n\n".join([
description,
f"**File to review:** {markdown.markdown_escape(item['file_path'])}"
])
])
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", item["severity"])
finding.set_meta("legacy.file", item["file_path"])
endpoints = list()
if item["file_path"]:
endpoints.append(Endpoint(raw=item["file_path"]))
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
except: # pylint: disable=W0702
log.exception("Failed to parse findings")
def parse_findings(filename, scanner):
""" Parse findings """
# Parse JSON using legacy parser
findings = DependencyCheckParser(filename).items
# Make finding instances
for item in findings:
finding = SastFinding(
title=item["title"],
description=[
"\n\n".join([
"Vulnerable dependency was found. Please upgrade component or check that vulnerable functionality is not used.", # pylint: disable=C0301
markdown.markdown_escape(item["description"]),
f"**File to review:** {markdown.markdown_escape(item['file_path'])}"
])
] + item["steps_to_reproduce"])
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", item["severity"])
finding.set_meta("legacy.file", item["file_path"])
endpoints = list()
if item["file_path"]:
endpoints.append(
namedtuple("Endpoint", ["raw"])(raw=item["file_path"]))
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def get_project_priorities(jira_client, project, issue_type="Bug"):
""" Returns list of Jira priorities in project """
try:
meta = jira_client.createmeta(projectKeys=project,
issuetypeNames=issue_type,
expand="projects.issuetypes.fields")
logging.debug("Got metadata for %d projects", len(meta["projects"]))
if not meta["projects"]:
logging.error("No meta returned for %s with type %s", project,
issue_type)
return []
project_meta = meta["projects"][0]
logging.debug("Got metadata for %d issuetypes",
len(project_meta["issuetypes"]))
if not project_meta["issuetypes"]:
logging.error("No %s in %s", issue_type, project)
return []
issue_types = project_meta["issuetypes"][0]
if "priority" not in issue_types["fields"]:
logging.error("No priority field in %s", project)
return []
priorities = [
priority["name"] for priority in \
issue_types["fields"]["priority"]["allowedValues"]
]
return priorities
except: # pylint: disable=W0702
logging.exception("Failed to get meta for %s", project)
return []
def execute(self):
""" Run the scanner """
# Prepare parameters
target_url = url.parse_url(self.config.get("target"))
nikto_parameters = shlex.split(self.config.get("nikto_parameters", ""))
# Make temporary files
output_file_fd, output_file = tempfile.mkstemp()
log.debug("Output file: %s", output_file)
os.close(output_file_fd)
# Prepare -Save option if needed
save_findings = list()
if self.config.get("save_intermediates_to", None):
base = os.path.join(self.config.get("save_intermediates_to"), __name__.split(".")[-2])
try:
os.makedirs(base, mode=0o755, exist_ok=True)
save_findings.append("-Save")
save_findings.append(base)
except:
pass
# Run scanner
task = subprocess.run(["perl", "nikto.pl"] + nikto_parameters + [
"-h", target_url.hostname, "-p", url.get_port(target_url),
"-Format", "xml", "-output", output_file
] + save_findings, cwd="/opt/nikto/program", stdout=subprocess.PIPE, stderr=subprocess.PIPE)
log.log_subprocess_result(task)
# Parse findings
parse_findings(output_file, self)
# Save intermediates
self.save_intermediates(output_file, task)
# Remove temporary files
os.remove(output_file)
def prepare_jira_mapping(jira_service):
""" Make Jira mapping (for projects that are using custom values) """
if not jira_service or not jira_service.valid:
return dict()
jira_service.connect()
issue_type = "Bug"
if jira_service.fields["issuetype"]["name"] != "!default_issuetype":
issue_type = jira_service.fields["issuetype"]["name"]
project_priorities = get_project_priorities(jira_service.client,
jira_service.project,
issue_type)
if not project_priorities:
jira_service.client.close()
return dict()
logging.debug("%s %s priorities: %s", jira_service.project, issue_type,
str(project_priorities))
c = const
mapping = dict()
for severity in c.JIRA_SEVERITIES:
if severity not in project_priorities:
for alternative in c.JIRA_ALTERNATIVES[severity]:
if alternative in project_priorities:
logging.warning("Mapping %s %s Jira priority: %s -> %s",
jira_service.project, issue_type, severity,
alternative)
mapping[severity] = alternative
break
if severity not in mapping:
mapping[severity] = project_priorities[0]
logging.error(
"Failed to find Jira mapping for %s, using %s as a fallback",
severity, mapping[severity])
jira_service.client.close()
return mapping
def parse_findings(filename, scanner):
""" Parse findings """
# Parse JSON using legacy parser
findings = SpotbugsParser(filename).items
# Make finding instances
for item in findings:
finding = SastFinding(
title=item["title"],
description=[
"\n\n".join([
item["description"],
f"**Category:** {markdown.markdown_escape(item['category'])}",
f"**File to review:** {markdown.markdown_escape(item['file_path'])}" \
f":{item['line']}"
])
] + item["steps_to_reproduce"]
)
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", constants.SPOTBUGS_SEVERITIES[item["severity"]])
finding.set_meta("legacy.file", item["file_path"])
finding.set_meta("legacy.line", item["line"])
endpoints = list()
if item["file_path"]:
endpoints.append(namedtuple("Endpoint", ["raw"])(raw=item["file_path"]))
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def parse_findings(output_file, scanner): # pylint: disable=E,W,R,C
""" Parse findings (code from dusty 1.0) """
# Parse HTML report using legacy parser
filtered_statuses = scanner.config.get(
"filtered_statuses", constants.PTAI_DEFAULT_FILTERED_STATUSES)
if isinstance(filtered_statuses, str):
filtered_statuses = [
item.strip() for item in filtered_statuses.split(",")
]
findings = PTAIScanParser(output_file, filtered_statuses).items
for item in findings:
finding = SastFinding(
title=item["title"],
description=[
html.escape(
markdown.markdown_escape(item["description"].replace(
" ", ""))) +
f"\n\n**File to review:** {markdown.markdown_escape(item['file_path'])}"
] + [html.escape(data) for data in item["steps_to_reproduce"]])
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity",
constants.PTAI_SEVERITIES[item["severity"]])
finding.set_meta("legacy.file", item["file_path"])
finding.set_meta(
"endpoints",
[namedtuple("Endpoint", ["raw"])(raw=item["file_path"])])
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def _setup_scan_policy(self):
self._scan_policy_name = "Default Policy"
# Use user-provided policy (if any)
if self.config.get("scan_policy_data", None) or self.config.get(
"scan_policy_from", None):
log.info("Using user-provided scan policy")
# Write to temp file if needed
if self.config.get("scan_policy_data", None):
policy_file_fd, policy_file = tempfile.mkstemp()
os.close(policy_file_fd)
with open(policy_file, "w") as policy:
log.debug("Scan policy data: '%s'",
self.config.get("scan_policy_data"))
policy.write(self.config.get("scan_policy_data"))
else:
policy_file = self.config.get("scan_policy_from")
# Load policy into ZAP
default_policies = self._zap_api.ascan.scan_policy_names
log.info("Importing scan policy from %s", policy_file)
self._zap_api.ascan.import_scan_policy(policy_file)
current_policies = self._zap_api.ascan.scan_policy_names
log.debug("Policies after load: %s", current_policies)
# Remove temporary file
if self.config.get("scan_policy_data", None):
os.remove(policy_file)
# Set name
loaded_policy_names = list(
set(current_policies) - set(default_policies))
if loaded_policy_names:
self._scan_policy_name = loaded_policy_names[0]
log.info("Scan policy set to '%s'", self._scan_policy_name)
return
# Setup 'simple' scan policy
self._scan_policies = [
item.strip()
for item in self.config.get("scan_types", "all").split(",")
]
# Disable globally blacklisted rules
for item in constants.ZAP_BLACKLISTED_RULES:
self._zap_api.ascan.set_scanner_alert_threshold(
id=item,
alertthreshold="OFF",
scanpolicyname=self._scan_policy_name)
self._zap_api.pscan.set_scanner_alert_threshold(
id=item, alertthreshold="OFF")
if "all" not in self._scan_policies:
# Disable all scanners first
for item in self._zap_api.ascan.scanners(self._scan_policy_name):
self._zap_api.ascan.set_scanner_alert_threshold(
id=item["id"],
alertthreshold="OFF",
scanpolicyname=self._scan_policy_name)
# Enable scanners from suite
for policy in self._scan_policies:
for item in constants.ZAP_SCAN_POCILICES.get(policy, []):
self._zap_api.ascan.set_scanner_alert_threshold(
id=item,
alertthreshold="DEFAULT",
scanpolicyname=self._scan_policy_name)
def parse_findings(filename, scanner):
""" Parse findings """
# Load JSON
try:
with open(filename, "r") as file:
data = json.load(file)
except: # pylint: disable=W0702
log.exception("Failed to load report JSON")
return
# Load CWE map
cwe_map = json.loads(
pkg_resources.resource_string(
"dusty",
f"{'/'.join(__name__.split('.')[1:-1])}/data/cwe_map_v4.2.json"))
# Parse JSON
if not isinstance(data, dict) or "vulnerabilities" not in data:
log.info("No data in report")
return
# Make finding instances
for item in data["vulnerabilities"]:
vuln_severity = cvss_to_severity(item.get("cvss", 0.0))
vuln_cwe = item.get("cwe", "Vulnerability")
#
vuln_cwe_title = cwe_map[vuln_cwe] if vuln_cwe in cwe_map else vuln_cwe
vuln_file_title = f" in {item.get('classMessage')}" if "classMessage" in item else ""
vuln_title = f"{vuln_cwe_title}{vuln_file_title}"
#
vuln_file = item.get("classMessage", "").rsplit(" (", 1)[0]
#
vuln_info_chunks = list()
if "longMessage" in item:
vuln_info_chunks.append(
markdown.markdown_escape(item["longMessage"]))
if "shortMessage" in item:
vuln_info_chunks.append(
markdown.markdown_escape(item["shortMessage"]))
vuln_info_chunks.append(
f"**Class:** {markdown.markdown_escape(item['classMessage'])}")
vuln_info_chunks.append(
f"**Method:** {markdown.markdown_escape(item['method'])}")
if "affectedFiles" in item:
vuln_info_chunks.append(
f"**Files:** {markdown.markdown_escape(', '.join(item['affectedFiles']))}"
)
#
finding = SastFinding(title=vuln_title,
description=["\n\n".join(vuln_info_chunks)])
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", vuln_severity)
finding.set_meta("legacy.file", vuln_file)
endpoints = list()
if vuln_file:
endpoints.append(namedtuple("Endpoint", ["raw"])(raw=vuln_file))
finding.set_meta("endpoints", endpoints)
log.debug(f"Endpoints: {finding.get_meta('endpoints')}")
scanner.findings.append(finding)
def load(self, config_seed, config_variable, config_file, suite):
""" Load and parse config """
config = self._load_config(config_seed, config_variable, config_file)
if not self._validate_config_base(config, suite):
raise ValueError("Invalid config")
context_config = self._prepare_context_config(config, suite)
self.context.suite = suite
self.context.config = context_config
log.debug("Resulting context config: %s", self.context.config)
log.info("Loaded %s suite configuration", self.context.suite)
def html_to_text(html, escape=True):
""" Convert HTML to markdown """
try:
text = inscriptis.get_text(html, display_links=True)
except: # pylint: disable=W0702
log.debug("Exception during HTML to text conversion\n%s",
traceback.format_exc())
text = ""
if escape:
text = markdown_escape(text)
return text
def execute(self, args):
""" Run the command """
log.debug("Starting")
if args.call_from_legacy:
log.warning("Called from legacy entry point")
# Init context
context = RunContext(args)
config = ConfigModel(context)
if args.list_suites or not args.suite:
suites = config.list_suites(args.config_seed, args.config_variable,
args.config_file)
if not args.suite:
log.error("Suite is not defined. Use --help to get help")
log.info("Available suites: %s", ", ".join(suites))
return
# Make instances
scanning = ScanningPerformer(context)
processing = ProcessingPerformer(context)
reporting = ReportingPerformer(context)
# Add to context
context.performers["scanning"] = scanning
context.performers["processing"] = processing
context.performers["reporting"] = reporting
# Init config
config.load(args.config_seed, args.config_variable, args.config_file,
args.suite)
scanning.validate_config(context.config)
processing.validate_config(context.config)
reporting.validate_config(context.config)
# Add meta to context
self._fill_context_meta(context)
# Load state
context.state.load()
# Prepare reporters first
reporting.prepare()
# Run actions
actions.run(context)
# Prepare scanning and processing
scanning.prepare()
processing.prepare()
# Perform
scanning.perform()
processing.perform()
reporting.perform()
# Done
context.state.save()
reporting.flush()
log.debug("Done")
# Show quality gate statistics if any
for line in context.get_meta("quality_gate_stats", list()):
log.info(line)
# Fail quality gate if needed
if context.get_meta("fail_quality_gate", False):
os._exit(1) # pylint: disable=W0212
def get_object(self, key):
""" Get object by key """
try:
log.debug("Trying to get object '%s' (path = '%s')", key,
self.config.get("path"))
with open(os.path.join(self.config.get("path"), key),
"rb") as file:
data = file.read()
return data
except: # pylint: disable=W0702
log.debug("Got exception: %s", traceback.format_exc())
return None
def main():
""" Main """
# Initialize argument parser
parent = argparse.ArgumentParser(add_help=False)
parent.add_argument("-d",
"--debug",
dest="log_level",
help="enable debug output",
action="store_const",
const=DEBUG,
default=INFO)
parent.add_argument("--call-from-legacy",
dest="call_from_legacy",
help=argparse.SUPPRESS,
action="store_true",
default=False)
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparsers = parser.add_subparsers(
dest="command",
title="commands",
help="command to execute, use <command> -h to get command help")
subparsers.required = True
# Load commands
commands = dict()
for _, name, pkg in pkgutil.iter_modules(dusty.commands.__path__):
if pkg:
continue
module = importlib.import_module("dusty.commands.{}".format(name))
argparser = subparsers.add_parser(
module.Command.get_name(),
parents=[parent],
help=module.Command.get_description(),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
commands[module.Command.get_name()] = module.Command(argparser)
# Parse arguments
args = parser.parse_args()
# Setup logging
log.init(args.log_level)
# Display welcome message
log.info(f"Dusty {pkg_resources.require('dusty')[0].version} is starting")
log.debug("Loaded commands: {}".format(", ".join(list(commands.keys()))))
# Install interrupt signal handler
signal.signal(signal.SIGINT, interrupt_handler)
# Run selected command
try:
commands[args.command].execute(args)
except:
log.exception("Command failed")
os._exit(1) # pylint: disable=W0212
# Display bye-bye message
log.info("All done. Have a nice day")
def _request_raw(self, endpoint, json=None):
""" Perform API request (directly) """
api = self._connection
if json is None:
response = api.get(f"{self.server}{endpoint}",
timeout=self.timeout)
else:
response = api.post(f"{self.server}{endpoint}",
json=json,
timeout=self.timeout)
log.debug("API response: %d [%s] %s", response.status_code,
response.headers, response.text)
return response
def parse_findings(output_file, scanner):
""" Parse findings (code from dusty 1.0) """
log.debug("Parsing findings")
# Load JSON
with open(output_file, "rb") as json_file:
data = json.load(json_file)
# Walk results
for issue in data:
title = f'Open port {issue["ports"][0]["port"]} found on {issue["ip"]}'
finding = DastFinding(title=title,
description=markdown.markdown_escape(title))
finding.set_meta("tool", scanner.get_name())
finding.set_meta("severity", SEVERITIES[-1])
scanner.findings.append(finding)
def execute(self):
""" Run the scanner """
log.debug(f"Config: {self.config}")
try:
self._start_zap()
if not self._wait_for_zap_start():
log.error("ZAP failed to start")
self.errors.append("ZAP daemon failed to start")
return
log.info("Target: %s", self.config.get("target"))
except BaseException as exception:
log.exception("Exception during ZAP scanning")
self.errors.append(str(exception))
finally:
self._stop_zap()
def execute(self):
""" Run the scanner """
# Make temporary directory
output_dir = tempfile.mkdtemp()
log.debug("Output directory: %s", output_dir)
# Run task
task = subprocess.run([
"insider", "-force", "-no-html",
"-target", self.config.get("code"), "-tech", self.config.get("tech")
], cwd=output_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
log.log_subprocess_result(task)
output_file = os.path.join(output_dir, os.listdir(output_dir)[0])
parse_findings(output_file, self)
# Save intermediates
self.save_intermediates(output_file, task)
def execute(self):
""" Run the scanner """
# Make temporary directory
output_dir = tempfile.mkdtemp()
log.debug("Output directory: %s", output_dir)
output_file = os.path.join(output_dir, "retirejs.json")
# Run task
task = subprocess.run([
"retire", f"--jspath={self.config.get('code')}", "--outputformat=json",
f"--outputpath={output_file}", "--includemeta", "--exitwith=0"
], cwd=output_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
log.log_subprocess_result(task)
parse_findings(output_file, self)
# Save intermediates
self.save_intermediates(output_file, task)
def report(self):
""" Report """
# Summary
for item in self.context.findings:
if item.get_meta("false_positive_finding", False) or item.get_meta("information_finding", False) or \
item.get_meta("excluded_finding", False):
continue
details = ''
if isinstance(item, DastFinding):
details = markdown.markdown_to_html(item.description)
elif isinstance(item, SastFinding):
details = markdown.markdown_to_html("<br/>".join(item.description))
log.debug(self.ado.create_finding(item.title, details, item.get_meta("severity", SEVERITIES[-1]),
assignee=self.assignee, issue_hash=item.get_meta("issue_hash", "")))
log.info("Creating findings")
def execute(self):
""" Run the scanner """
# Get target host IP
target_url = url.parse_url(self.config.get("target"))
host = target_url.hostname
if not url.find_ip(host):
task = subprocess.run(["getent", "hosts", host],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.log_subprocess_result(task)
host = url.find_ip(task.stdout.decode("utf-8", errors="ignore"))
if host:
host = host[0].strip()
if not host:
log.warning("No target IP found. Exiting")
return
# Prepare config
include_ports = list()
if self.config.get("include_ports", "0-65535"):
include_ports.append("-p")
include_ports.append(
f'{self.config.get("include_ports", "0-65535")}')
include_ports.append(
f'-pU:{self.config.get("include_ports", "0-65535")}')
exclude_ports = list()
if self.config.get("exclude_ports", None):
exclude_ports.append("--exclude-ports")
exclude_ports.append(f'{self.config.get("exclude_ports")}')
# Make temporary files
output_file_fd, output_file = tempfile.mkstemp()
log.debug("Output file: %s", output_file)
os.close(output_file_fd)
# Scan target
task = subprocess.run(["masscan", host] + include_ports + [
"--rate",
"1000",
"-oJ",
output_file,
] + exclude_ports,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.log_subprocess_result(task)
# Parse findings
parse_findings(output_file, self)
# Save intermediates
self.save_intermediates(output_file, task)
# Remove temporary files
os.remove(output_file)
