def requires(self):
""" GatherWebTargets depends on ParseMasscanOutput and ParseAmassOutput to run.
ParseMasscanOutput expects rate, target_file, interface, and either ports or top_ports as parameters.
ParseAmassOutput accepts exempt_list and expects target_file
Returns:
dict(str: ParseMasscanOutput, str: ParseAmassOutput)
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
}
return {
"masscan-output":
ParseMasscanOutput(**args),
"amass-output":
ParseAmassOutput(exempt_list=self.exempt_list,
target_file=self.target_file,
results_dir=self.results_dir),
}
def test_parse_amass_output_locations(tmp_path):
pao = ParseAmassOutput(target_file=tf, exempt_list=el, results_dir=str(tmp_path))
assert pao.output().get("target-ips").path == str((Path(tmp_path) / "target-results" / "ipv4_addresses").resolve())
assert pao.output().get("target-ip6s").path == str((Path(tmp_path) / "target-results" / "ipv6_addresses").resolve())
assert pao.output().get("target-subdomains").path == str(
(Path(tmp_path) / "target-results" / "subdomains").resolve()
)
def test_parse_amass_ip6_results(tmp_path):
pao = ParseAmassOutput(target_file=tf, exempt_list=el, results_dir=str(tmp_path))
pao.input = lambda: luigi.LocalTarget(amass_json)
pao.run()
contents = (Path(pao.output().get("target-ip6s").path)).read_text()
for line in contents.split():
assert line.strip() in ip6s
def test_parse_amass_ip6_results_only_contain_ipv6_addys(tmp_path):
pao = ParseAmassOutput(target_file=tf, exempt_list=el, results_dir=str(tmp_path))
pao.input = lambda: luigi.LocalTarget(amass_json)
pao.run()
contents = (Path(pao.output().get("target-ip6s").path)).read_text()
for line in contents.split():
try:
ipaddress.ip_interface(line.strip()) # is it a valid ip/network?
except ValueError:
assert 0
def run(self):
""" Defines the options/arguments sent to masscan after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
if self.ports and self.top_ports:
# can't have both
logging.error(
"Only --ports or --top-ports is permitted, not both.")
exit(1)
if not self.ports and not self.top_ports:
# need at least one
logging.error("Must specify either --top-ports or --ports.")
exit(2)
if self.top_ports < 0:
# sanity check
logging.error("--top-ports must be greater than 0")
exit(3)
if self.top_ports:
# if --top-ports used, format the top_*_ports lists as strings and then into a proper masscan --ports option
top_tcp_ports_str = ",".join(
str(x) for x in top_tcp_ports[:self.top_ports])
top_udp_ports_str = ",".join(
str(x) for x in top_udp_ports[:self.top_ports])
self.ports = f"{top_tcp_ports_str},U:{top_udp_ports_str}"
self.top_ports = 0
target_list = yield TargetList(target_file=self.target_file,
results_dir=self.results_dir)
Path(self.output().path).parent.mkdir(parents=True, exist_ok=True)
Path(self.output().path).parent.mkdir(parents=True, exist_ok=True)
if target_list.path.endswith("domains"):
yield ParseAmassOutput(target_file=self.target_file,
exempt_list=self.exempt_list,
results_dir=self.results_dir)
command = [
"masscan",
"-v",
"--open",
"--banners",
"--rate",
self.rate,
"-e",
self.interface,
"-oJ",
self.output().path,
"--ports",
self.ports,
"-iL",
]
if target_list.path.endswith("domains"):
command.append(
target_list.path.replace("domains", "ipv4_addresses"))
else:
command.append(target_list.path.replace("domains", "ip_addresses"))
subprocess.run(command)