def test_install_masscan():
masscan = Path(tool_paths.get("masscan"))
setup_install_test(masscan)
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install masscan")
assert masscan.exists()
def test_install_aquatone():
aquatone = Path(tool_paths.get("aquatone"))
setup_install_test(aquatone)
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install aquatone")
assert aquatone.exists()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.sentry = False
self.prompt = "recon-pipeline> "
self.selectorloop = None
self.continue_install = True
Path(defaults.get("tools-dir")).mkdir(parents=True, exist_ok=True)
# register hooks to handle selector loop start and cleanup
self.register_preloop_hook(self._preloop_hook)
self.register_postloop_hook(self._postloop_hook)
class TargetList(luigi.ExternalTask):
""" External task. ``TARGET_FILE`` is generated manually by the user from target's scope.
Args:
results_dir: specifies the directory on disk to which all Task results are written
"""
target_file = luigi.Parameter()
results_dir = luigi.Parameter(default=defaults.get("results-dir", ""))
def output(self):
""" Returns the target output for this task. target_file.ips || target_file.domains
In this case, it expects a file to be present in the local filesystem.
By convention, TARGET_NAME should be something like tesla or some other
target identifier. The returned target output will either be target_file.ips
or target_file.domains, depending on what is found on the first line of the file.
Example: Given a TARGET_FILE of tesla where the first line is tesla.com; tesla.domains
is written to disk.
Returns:
luigi.local_target.LocalTarget
"""
self.results_dir = Path(self.results_dir)
self.target_file = Path(self.target_file)
try:
with open(self.target_file) as f:
first_line = f.readline()
ipaddress.ip_interface(
first_line.strip()) # is it a valid ip/network?
except OSError as e:
# can't open file; log error / return nothing
return logging.error(f"opening {self.target_file}: {e.strerror}")
except ValueError as e:
# exception thrown by ip_interface; domain name assumed
logging.debug(e)
new_target = "domains"
else:
# no exception thrown; ip address found
new_target = "ip_addresses"
results_subfolder = self.results_dir / "target-results"
results_subfolder.mkdir(parents=True, exist_ok=True)
new_path = results_subfolder / new_target
shutil.copy(self.target_file, new_path.resolve())
return luigi.LocalTarget(new_path.resolve())
def test_install_luigi():
setup_install_test()
if shutil.which("luigi") is not None:
subprocess.run("pip uninstall luigi".split())
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install luigi")
assert shutil.which("luigi") is not None
def test_install_corscanner():
corscanner = Path(tool_paths.get("CORScanner"))
setup_install_test(corscanner)
if corscanner.parent.exists():
shutil.rmtree(corscanner.parent)
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install corscanner")
assert corscanner.exists()
def test_install_recursive_gobuster():
recursive_gobuster = Path(tool_paths.get("recursive-gobuster"))
setup_install_test(recursive_gobuster)
if recursive_gobuster.parent.exists():
shutil.rmtree(recursive_gobuster.parent)
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install recursive-gobuster")
assert recursive_gobuster.exists()
def test_update_recursive_gobuster():
recursive_gobuster = Path(tool_paths.get("recursive-gobuster"))
setup_install_test()
if not recursive_gobuster.parent.exists():
subprocess.run(
f"git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}"
.split())
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install recursive-gobuster")
assert recursive_gobuster.exists()
def test_update_corscanner():
corscanner = Path(tool_paths.get("CORScanner"))
setup_install_test()
if not corscanner.parent.exists():
subprocess.run(
f"git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}"
.split())
rs = recon_pipeline.ReconShell()
assert Path(defaults.get("tools-dir")).exists()
run_cmd(rs, "install corscanner")
assert corscanner.exists()
class WebanalyzeScan(luigi.Task):
""" Use webanalyze to determine the technology stack on the given target(s).
Install:
.. code-block:: console
go get -u github.com/rverton/webanalyze
# loads new apps.json file from wappalyzer project
webanalyze -update
Basic Example:
.. code-block:: console
webanalyze -host www.tesla.com -output json
Luigi Example:
.. code-block:: console
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.webanalyze WebanalyzeScan --target-file tesla --top-ports 1000 --interface eth0
Args:
threads: number of threads for parallel webanalyze command execution
exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
top_ports: Scan top N most popular ports *--* Required by upstream Task
ports: specifies the port(s) to be scanned *--* Required by upstream Task
interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
def requires(self):
""" WebanalyzeScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
The naming convention for the output file is webanalyze.TARGET_FILE.txt
Results are stored in their own directory: webanalyze-TARGET_FILE-results
Returns:
luigi.local_target.LocalTarget
"""
results_subfolder = Path(self.results_dir) / "webanalyze-results"
return luigi.LocalTarget(results_subfolder.resolve())
def _wrapped_subprocess(self, cmd):
with open(
f"webanalyze.{cmd[2].replace('//', '_').replace(':', '')}.txt",
"wb") as f:
subprocess.run(cmd, stderr=f)
def run(self):
""" Defines the options/arguments sent to webanalyze after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
try:
self.threads = abs(int(self.threads))
except TypeError:
return logging.error(
"The value supplied to --threads must be a non-negative integer."
)
commands = list()
with self.input().open() as f:
for target in f:
target = target.strip()
try:
if isinstance(ipaddress.ip_address(target),
ipaddress.IPv6Address): # ipv6
target = f"[{target}]"
except ValueError:
# domain names raise ValueErrors, just assume we have a domain and keep on keepin on
pass
for url_scheme in ("https://", "http://"):
command = [
tool_paths.get("webanalyze"), "-host",
f"{url_scheme}{target}"
]
commands.append(command)
Path(self.output().path).mkdir(parents=True, exist_ok=True)
cwd = Path().cwd()
os.chdir(self.output().path)
if not Path("apps.json").exists():
subprocess.run(f"{tool_paths.get('webanalyze')} -update".split())
with ThreadPoolExecutor(max_workers=self.threads) as executor:
executor.map(self._wrapped_subprocess, commands)
os.chdir(str(cwd))
class AquatoneScan(luigi.Task):
""" Screenshot all web targets and generate HTML report.
aquatone commands are structured like the example below.
cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20
An example of the corresponding luigi command is shown below.
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.aquatone AquatoneScan --target-file tesla --top-ports 1000
Args:
threads: number of threads for parallel aquatone command execution
scan_timeout: timeout in miliseconds for aquatone port scans
exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
top_ports: Scan top N most popular ports *--* Required by upstream Task
ports: specifies the port(s) to be scanned *--* Required by upstream Task
interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
scan_timeout = luigi.Parameter(default=defaults.get("aquatone-scan-timeout", ""))
def requires(self):
""" AquatoneScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
Naming convention for the output file is amass.TARGET_FILE.json.
Returns:
luigi.local_target.LocalTarget
"""
return luigi.LocalTarget(f"{self.results_dir}/aquatone-{self.target_file}-results")
def run(self):
""" Defines the options/arguments sent to aquatone after processing.
cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
Path(self.output().path).mkdir(parents=True, exist_ok=True)
command = [
tool_paths.get("aquatone"),
"-scan-timeout",
self.scan_timeout,
"-threads",
self.threads,
"-silent",
"-out",
self.output().path,
]
with self.input().open() as target_list:
subprocess.run(command, stdin=target_list)
class SubjackScan(ExternalProgramTask):
""" Use subjack to scan for potential subdomain takeovers.
subjack commands are structured like the example below.
subjack -w webtargets.tesla.txt -t 100 -timeout 30 -o subjack.tesla.txt -ssl
An example of the corresponding luigi command is shown below.
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.subdomain_takeover SubjackScan --target-file tesla --top-ports 1000 --interface eth0
Args:
threads: number of threads for parallel subjack command execution
exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
top_ports: Scan top N most popular ports *--* Required by upstream Task
ports: specifies the port(s) to be scanned *--* Required by upstream Task
interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
def requires(self):
""" SubjackScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
Naming convention for the output file is subjack.TARGET_FILE.txt.
Returns:
luigi.local_target.LocalTarget
"""
return luigi.LocalTarget(
f"{self.results_dir}/subjack.{self.target_file}.txt")
def program_args(self):
""" Defines the options/arguments sent to subjack after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
command = [
tool_paths.get("subjack"),
"-w",
self.input().path,
"-t",
self.threads,
"-a",
"-timeout",
"30",
"-o",
self.output().path,
"-v",
"-ssl",
"-c",
tool_paths.get("subjack-fingerprints"),
]
return command
def test_wordlist_exists():
if is_kali():
assert Path(defaults.get("gobuster-wordlist")).exists()
def test_threads_numeric():
assert defaults.get("threads").isnumeric()
def test_masscan_rate_numeric():
assert defaults.get("masscan-rate").isnumeric()
class CORScannerScan(ExternalProgramTask):
""" Use CORScanner to scan for potential CORS misconfigurations.
CORScanner commands are structured like the example below.
python cors_scan.py -i webtargets.tesla.txt -t 100
An example of the corresponding luigi command is shown below.
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.corscanner CORScannerScan --target-file tesla --top-ports 1000 --interface eth0
Install:
git clone https://github.com/chenjj/CORScanner.git
cd CORScanner
pip install -r requirements.txt
pip install future
Args:
threads: number of threads for parallel subjack command execution
exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
top_ports: Scan top N most popular ports *--* Required by upstream Task
ports: specifies the port(s) to be scanned *--* Required by upstream Task
interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
def requires(self):
""" CORScannerScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
Naming convention for the output file is corscanner.TARGET_FILE.json.
Returns:
luigi.local_target.LocalTarget
"""
return luigi.LocalTarget(
f"{self.results_dir}/corscanner.{self.target_file}.json")
def program_args(self):
""" Defines the options/arguments sent to tko-subs after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
command = [
"python3",
tool_paths.get("CORScanner"),
"-i",
self.input().path,
"-t",
self.threads,
"-o",
self.output().path,
]
return command
class ThreadedNmapScan(luigi.Task):
""" Run ``nmap`` against specific targets and ports gained from the ParseMasscanOutput Task.
Install:
``nmap`` is already on your system if you're using kali. If you're not using kali, refer to your own
distributions instructions for installing ``nmap``.
Basic Example:
.. code-block:: console
nmap --open -sT -sC -T 4 -sV -Pn -p 43,25,21,53,22 -oA htb-targets-nmap-results/nmap.10.10.10.155-tcp 10.10.10.155
Luigi Example:
.. code-block:: console
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.nmap ThreadedNmap --target-file htb-targets --top-ports 5000
Args:
threads: number of threads for parallel nmap command execution
rate: desired rate for transmitting packets (packets per second) *Required by upstream Task*
interface: use the named raw network interface, such as "eth0" *Required by upstream Task*
top_ports: Scan top N most popular ports *Required by upstream Task*
ports: specifies the port(s) to be scanned *Required by upstream Task*
target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task*
results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task*
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
def requires(self):
""" ThreadedNmap depends on ParseMasscanOutput to run.
TargetList expects target_file as a parameter.
Masscan expects rate, target_file, interface, and either ports or top_ports as parameters.
Returns:
luigi.Task - ParseMasscanOutput
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
}
return ParseMasscanOutput(**args)
def output(self):
""" Returns the target output for this task.
Naming convention for the output folder is TARGET_FILE-nmap-results.
The output folder will be populated with all of the output files generated by
any nmap commands run. Because the nmap command uses -oA, there will be three
files per target scanned: .xml, .nmap, .gnmap.
Returns:
luigi.local_target.LocalTarget
"""
results_subfolder = Path(self.results_dir) / "nmap-results"
return luigi.LocalTarget(results_subfolder.resolve())
def run(self):
""" Parses pickled target info dictionary and runs targeted nmap scans against only open ports. """
try:
self.threads = abs(int(self.threads))
except TypeError:
return logging.error(
"The value supplied to --threads must be a non-negative integer."
)
ip_dict = pickle.load(open(self.input().path, "rb"))
nmap_command = [ # placeholders will be overwritten with appropriate info in loop below
"nmap",
"--open",
"PLACEHOLDER-IDX-2",
"-n",
"-sC",
"-T",
"4",
"-sV",
"-Pn",
"-p",
"PLACEHOLDER-IDX-10",
"-oA",
]
commands = list()
"""
ip_dict structure
{
"IP_ADDRESS":
{'udp': {"161", "5000", ... },
...
i.e. {protocol: set(ports) }
}
"""
for target, protocol_dict in ip_dict.items():
for protocol, ports in protocol_dict.items():
tmp_cmd = nmap_command[:]
tmp_cmd[2] = "-sT" if protocol == "tcp" else "-sU"
# arg to -oA, will drop into subdir off curdir
tmp_cmd[10] = ",".join(ports)
tmp_cmd.append(
str(
Path(self.output().path) /
f"nmap.{target}-{protocol}"))
tmp_cmd.append(target) # target as final arg to nmap
commands.append(tmp_cmd)
# basically mkdir -p, won't error out if already there
Path(self.output().path).mkdir(parents=True, exist_ok=True)
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.threads) as executor:
executor.map(subprocess.run, commands)
class GobusterScan(luigi.Task):
""" Use gobuster to perform forced browsing.
gobuster commands are structured like the example below.
gobuster dir -q -e -k -t 20 -u www.tesla.com -w /usr/share/seclists/Discovery/Web-Content/common.txt -p http://127.0.0.1:8080 -o gobuster.tesla.txt -x php,html
An example of the corresponding luigi command is shown below.
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.gobuster GobusterScan --target-file tesla --top-ports 1000 \
--interface eth0 --proxy http://127.0.0.1:8080 --extensions php,html \
--wordlist /usr/share/seclists/Discovery/Web-Content/common.txt --threads 20
Install:
go get github.com/OJ/gobuster
git clone https://github.com/epi052/recursive-gobuster.git
Args:
threads: number of threads for parallel gobuster command execution
wordlist: wordlist used for forced browsing
extensions: additional extensions to apply to each item in the wordlist
recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly)
exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task
top_ports: Scan top N most popular ports *--* Required by upstream Task
ports: specifies the port(s) to be scanned *--* Required by upstream Task
interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task
rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task
target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task
results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task
"""
proxy = luigi.Parameter(default=defaults.get("proxy", ""))
threads = luigi.Parameter(default=defaults.get("threads", ""))
wordlist = luigi.Parameter(default=defaults.get("gobuster-wordlist", ""))
extensions = luigi.Parameter(default=defaults.get("gobuster-extensions", ""))
recursive = luigi.BoolParameter(default=False)
def requires(self):
""" GobusterScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
If recursion is disabled, the naming convention for the output file is gobuster.TARGET_FILE.txt
Otherwise the output file is recursive-gobuster_TARGET_FILE.log
Results are stored in their own directory: gobuster-TARGET_FILE-results
Returns:
luigi.local_target.LocalTarget
"""
return luigi.LocalTarget(f"{self.results_dir}/gobuster-{self.target_file}-results")
def run(self):
""" Defines the options/arguments sent to gobuster after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
try:
self.threads = abs(int(self.threads))
except TypeError:
return logging.error("The value supplied to --threads must be a non-negative integer.")
commands = list()
with self.input().open() as f:
for target in f:
target = target.strip()
try:
if isinstance(ipaddress.ip_address(target), ipaddress.IPv6Address): # ipv6
target = f"[{target}]"
except ValueError:
# domain names raise ValueErrors, just assume we have a domain and keep on keepin on
pass
for url_scheme in ("https://", "http://"):
if self.recursive:
command = [
tool_paths.get("recursive-gobuster"),
"-w",
self.wordlist,
f"{url_scheme}{target}",
]
else:
command = [
tool_paths.get("gobuster"),
"dir",
"-q",
"-e",
"-k",
"-u",
f"{url_scheme}{target}",
"-w",
self.wordlist,
"-o",
Path(self.output().path).joinpath(
f"gobuster.{url_scheme.replace('//', '_').replace(':', '')}{target}.txt"
),
]
if self.extensions:
command.extend(["-x", self.extensions])
if self.proxy:
command.extend(["-p", self.proxy])
commands.append(command)
Path(self.output().path).mkdir(parents=True, exist_ok=True)
if self.recursive:
# workaround for recursive gobuster not accepting output directory
cwd = Path().cwd()
os.chdir(self.output().path)
with ThreadPoolExecutor(max_workers=self.threads) as executor:
executor.map(subprocess.run, commands)
if self.recursive:
os.chdir(str(cwd))
class AquatoneScan(luigi.Task):
""" Screenshot all web targets and generate HTML report.
Install:
.. code-block:: console
mkdir /tmp/aquatone
wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip
unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone
sudo mv /tmp/aquatone/aquatone /usr/local/bin/aquatone
rm -rf /tmp/aquatone
Basic Example:
``aquatone`` commands are structured like the example below.
``cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20``
Luigi Example:
.. code-block:: python
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.aquatone AquatoneScan --target-file tesla --top-ports 1000
Args:
threads: number of threads for parallel aquatone command execution
scan_timeout: timeout in miliseconds for aquatone port scans
exempt_list: Path to a file providing blacklisted subdomains, one per line. *Optional by upstream Task*
top_ports: Scan top N most popular ports *Required by upstream Task*
ports: specifies the port(s) to be scanned *Required by upstream Task*
interface: use the named raw network interface, such as "eth0" *Required by upstream Task*
rate: desired rate for transmitting packets (packets per second) *Required by upstream Task*
target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task*
results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task*
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
scan_timeout = luigi.Parameter(
default=defaults.get("aquatone-scan-timeout", ""))
def requires(self):
""" AquatoneScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
Naming convention for the output file is amass.TARGET_FILE.json.
Returns:
luigi.local_target.LocalTarget
"""
results_subfolder = Path(self.results_dir) / "aquatone-results"
return luigi.LocalTarget(results_subfolder.resolve())
def run(self):
""" Defines the options/arguments sent to aquatone after processing.
cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
Path(self.output().path).mkdir(parents=True, exist_ok=True)
command = [
tool_paths.get("aquatone"),
"-scan-timeout",
self.scan_timeout,
"-threads",
self.threads,
"-silent",
"-out",
self.output().path,
]
with self.input().open() as target_list:
subprocess.run(command, stdin=target_list)
def test_aquatone_scan_timeout_numeric():
assert defaults.get("aquatone-scan-timeout").isnumeric()
class MasscanScan(luigi.Task):
""" Run ``masscan`` against a target specified via the TargetList Task.
Note:
When specified, ``--top_ports`` is processed and then ultimately passed to ``--ports``.
Install:
.. code-block:: console
git clone https://github.com/robertdavidgraham/masscan /tmp/masscan
make -s -j -C /tmp/masscan
sudo mv /tmp/masscan/bin/masscan /usr/local/bin/masscan
rm -rf /tmp/masscan
Basic Example:
.. code-block:: console
masscan -v --open-only --banners --rate 1000 -e tun0 -oJ masscan.tesla.json --ports 80,443,22,21 -iL tesla.ips
Luigi Example:
.. code-block:: console
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.masscan Masscan --target-file tesla --ports 80,443,22,21
Args:
rate: desired rate for transmitting packets (packets per second)
interface: use the named raw network interface, such as "eth0"
top_ports: Scan top N most popular ports
ports: specifies the port(s) to be scanned
target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task*
results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task*
exempt_list: Path to a file providing blacklisted subdomains, one per line. *Optional by upstream Task*
"""
rate = luigi.Parameter(default=defaults.get("masscan-rate", ""))
interface = luigi.Parameter(default=defaults.get("masscan-iface", ""))
top_ports = luigi.IntParameter(
default=0) # IntParameter -> top_ports expected as int
ports = luigi.Parameter(default="")
def output(self):
""" Returns the target output for this task.
Naming convention for the output file is masscan.TARGET_FILE.json.
Returns:
luigi.local_target.LocalTarget
"""
results_subfolder = Path(self.results_dir) / "masscan-results"
new_path = results_subfolder / "masscan.json"
return luigi.LocalTarget(new_path.resolve())
def run(self):
""" Defines the options/arguments sent to masscan after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
if self.ports and self.top_ports:
# can't have both
logging.error(
"Only --ports or --top-ports is permitted, not both.")
exit(1)
if not self.ports and not self.top_ports:
# need at least one
logging.error("Must specify either --top-ports or --ports.")
exit(2)
if self.top_ports < 0:
# sanity check
logging.error("--top-ports must be greater than 0")
exit(3)
if self.top_ports:
# if --top-ports used, format the top_*_ports lists as strings and then into a proper masscan --ports option
top_tcp_ports_str = ",".join(
str(x) for x in top_tcp_ports[:self.top_ports])
top_udp_ports_str = ",".join(
str(x) for x in top_udp_ports[:self.top_ports])
self.ports = f"{top_tcp_ports_str},U:{top_udp_ports_str}"
self.top_ports = 0
target_list = yield TargetList(target_file=self.target_file,
results_dir=self.results_dir)
Path(self.output().path).parent.mkdir(parents=True, exist_ok=True)
Path(self.output().path).parent.mkdir(parents=True, exist_ok=True)
if target_list.path.endswith("domains"):
yield ParseAmassOutput(target_file=self.target_file,
exempt_list=self.exempt_list,
results_dir=self.results_dir)
command = [
"masscan",
"-v",
"--open",
"--banners",
"--rate",
self.rate,
"-e",
self.interface,
"-oJ",
self.output().path,
"--ports",
self.ports,
"-iL",
]
if target_list.path.endswith("domains"):
command.append(
target_list.path.replace("domains", "ipv4_addresses"))
else:
command.append(target_list.path.replace("domains", "ip_addresses"))
subprocess.run(command)
class SubjackScan(ExternalProgramTask):
""" Use ``subjack`` to scan for potential subdomain takeovers.
Install:
.. code-block:: console
go get github.com/haccer/subjack
cd ~/go/src/github.com/haccer/subjack
go build
go install
Basic Example:
.. code-block:: console
subjack -w webtargets.tesla.txt -t 100 -timeout 30 -o subjack.tesla.txt -ssl
Luigi Example:
.. code-block:: console
PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.subdomain_takeover SubjackScan --target-file tesla --top-ports 1000 --interface eth0
Args:
threads: number of threads for parallel subjack command execution
exempt_list: Path to a file providing blacklisted subdomains, one per line. *Optional by upstream Task*
top_ports: Scan top N most popular ports *Required by upstream Task*
ports: specifies the port(s) to be scanned *Required by upstream Task*
interface: use the named raw network interface, such as "eth0" *Required by upstream Task*
rate: desired rate for transmitting packets (packets per second) *Required by upstream Task*
target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task*
results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task*
"""
threads = luigi.Parameter(default=defaults.get("threads", ""))
def requires(self):
""" SubjackScan depends on GatherWebTargets to run.
GatherWebTargets accepts exempt_list and expects rate, target_file, interface,
and either ports or top_ports as parameters
Returns:
luigi.Task - GatherWebTargets
"""
args = {
"results_dir": self.results_dir,
"rate": self.rate,
"target_file": self.target_file,
"top_ports": self.top_ports,
"interface": self.interface,
"ports": self.ports,
"exempt_list": self.exempt_list,
}
return GatherWebTargets(**args)
def output(self):
""" Returns the target output for this task.
Naming convention for the output file is subjack.TARGET_FILE.txt.
Returns:
luigi.local_target.LocalTarget
"""
results_subfolder = Path(self.results_dir) / "subjack-results"
new_path = results_subfolder / "subjack.txt"
return luigi.LocalTarget(new_path.resolve())
def program_args(self):
""" Defines the options/arguments sent to subjack after processing.
Returns:
list: list of options/arguments, beginning with the name of the executable to run
"""
Path(self.output().path).parent.mkdir(parents=True, exist_ok=True)
command = [
tool_paths.get("subjack"),
"-w",
self.input().path,
"-t",
self.threads,
"-a",
"-timeout",
"30",
"-o",
self.output().path,
"-v",
"-ssl",
"-c",
tool_paths.get("subjack-fingerprints"),
]
return command
