def test_install_masscan(): masscan = Path(tool_paths.get("masscan")) setup_install_test(masscan) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install masscan") assert masscan.exists()
def test_install_aquatone(): aquatone = Path(tool_paths.get("aquatone")) setup_install_test(aquatone) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install aquatone") assert aquatone.exists()
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.sentry = False self.prompt = "recon-pipeline> " self.selectorloop = None self.continue_install = True Path(defaults.get("tools-dir")).mkdir(parents=True, exist_ok=True) # register hooks to handle selector loop start and cleanup self.register_preloop_hook(self._preloop_hook) self.register_postloop_hook(self._postloop_hook)
class TargetList(luigi.ExternalTask): """ External task. ``TARGET_FILE`` is generated manually by the user from target's scope. Args: results_dir: specifies the directory on disk to which all Task results are written """ target_file = luigi.Parameter() results_dir = luigi.Parameter(default=defaults.get("results-dir", "")) def output(self): """ Returns the target output for this task. target_file.ips || target_file.domains In this case, it expects a file to be present in the local filesystem. By convention, TARGET_NAME should be something like tesla or some other target identifier. The returned target output will either be target_file.ips or target_file.domains, depending on what is found on the first line of the file. Example: Given a TARGET_FILE of tesla where the first line is tesla.com; tesla.domains is written to disk. Returns: luigi.local_target.LocalTarget """ self.results_dir = Path(self.results_dir) self.target_file = Path(self.target_file) try: with open(self.target_file) as f: first_line = f.readline() ipaddress.ip_interface( first_line.strip()) # is it a valid ip/network? except OSError as e: # can't open file; log error / return nothing return logging.error(f"opening {self.target_file}: {e.strerror}") except ValueError as e: # exception thrown by ip_interface; domain name assumed logging.debug(e) new_target = "domains" else: # no exception thrown; ip address found new_target = "ip_addresses" results_subfolder = self.results_dir / "target-results" results_subfolder.mkdir(parents=True, exist_ok=True) new_path = results_subfolder / new_target shutil.copy(self.target_file, new_path.resolve()) return luigi.LocalTarget(new_path.resolve())
def test_install_luigi(): setup_install_test() if shutil.which("luigi") is not None: subprocess.run("pip uninstall luigi".split()) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install luigi") assert shutil.which("luigi") is not None
def test_install_corscanner(): corscanner = Path(tool_paths.get("CORScanner")) setup_install_test(corscanner) if corscanner.parent.exists(): shutil.rmtree(corscanner.parent) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install corscanner") assert corscanner.exists()
def test_install_recursive_gobuster(): recursive_gobuster = Path(tool_paths.get("recursive-gobuster")) setup_install_test(recursive_gobuster) if recursive_gobuster.parent.exists(): shutil.rmtree(recursive_gobuster.parent) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install recursive-gobuster") assert recursive_gobuster.exists()
def test_update_recursive_gobuster(): recursive_gobuster = Path(tool_paths.get("recursive-gobuster")) setup_install_test() if not recursive_gobuster.parent.exists(): subprocess.run( f"git clone https://github.com/epi052/recursive-gobuster.git {recursive_gobuster.parent}" .split()) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install recursive-gobuster") assert recursive_gobuster.exists()
def test_update_corscanner(): corscanner = Path(tool_paths.get("CORScanner")) setup_install_test() if not corscanner.parent.exists(): subprocess.run( f"git clone https://github.com/chenjj/CORScanner.git {corscanner.parent}" .split()) rs = recon_pipeline.ReconShell() assert Path(defaults.get("tools-dir")).exists() run_cmd(rs, "install corscanner") assert corscanner.exists()
class WebanalyzeScan(luigi.Task): """ Use webanalyze to determine the technology stack on the given target(s). Install: .. code-block:: console go get -u github.com/rverton/webanalyze # loads new apps.json file from wappalyzer project webanalyze -update Basic Example: .. code-block:: console webanalyze -host www.tesla.com -output json Luigi Example: .. code-block:: console PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.webanalyze WebanalyzeScan --target-file tesla --top-ports 1000 --interface eth0 Args: threads: number of threads for parallel webanalyze command execution exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task top_ports: Scan top N most popular ports *--* Required by upstream Task ports: specifies the port(s) to be scanned *--* Required by upstream Task interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task """ threads = luigi.Parameter(default=defaults.get("threads", "")) def requires(self): """ WebanalyzeScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. The naming convention for the output file is webanalyze.TARGET_FILE.txt Results are stored in their own directory: webanalyze-TARGET_FILE-results Returns: luigi.local_target.LocalTarget """ results_subfolder = Path(self.results_dir) / "webanalyze-results" return luigi.LocalTarget(results_subfolder.resolve()) def _wrapped_subprocess(self, cmd): with open( f"webanalyze.{cmd[2].replace('//', '_').replace(':', '')}.txt", "wb") as f: subprocess.run(cmd, stderr=f) def run(self): """ Defines the options/arguments sent to webanalyze after processing. Returns: list: list of options/arguments, beginning with the name of the executable to run """ try: self.threads = abs(int(self.threads)) except TypeError: return logging.error( "The value supplied to --threads must be a non-negative integer." ) commands = list() with self.input().open() as f: for target in f: target = target.strip() try: if isinstance(ipaddress.ip_address(target), ipaddress.IPv6Address): # ipv6 target = f"[{target}]" except ValueError: # domain names raise ValueErrors, just assume we have a domain and keep on keepin on pass for url_scheme in ("https://", "http://"): command = [ tool_paths.get("webanalyze"), "-host", f"{url_scheme}{target}" ] commands.append(command) Path(self.output().path).mkdir(parents=True, exist_ok=True) cwd = Path().cwd() os.chdir(self.output().path) if not Path("apps.json").exists(): subprocess.run(f"{tool_paths.get('webanalyze')} -update".split()) with ThreadPoolExecutor(max_workers=self.threads) as executor: executor.map(self._wrapped_subprocess, commands) os.chdir(str(cwd))
class AquatoneScan(luigi.Task): """ Screenshot all web targets and generate HTML report. aquatone commands are structured like the example below. cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20 An example of the corresponding luigi command is shown below. PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.aquatone AquatoneScan --target-file tesla --top-ports 1000 Args: threads: number of threads for parallel aquatone command execution scan_timeout: timeout in miliseconds for aquatone port scans exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task top_ports: Scan top N most popular ports *--* Required by upstream Task ports: specifies the port(s) to be scanned *--* Required by upstream Task interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task """ threads = luigi.Parameter(default=defaults.get("threads", "")) scan_timeout = luigi.Parameter(default=defaults.get("aquatone-scan-timeout", "")) def requires(self): """ AquatoneScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. Naming convention for the output file is amass.TARGET_FILE.json. Returns: luigi.local_target.LocalTarget """ return luigi.LocalTarget(f"{self.results_dir}/aquatone-{self.target_file}-results") def run(self): """ Defines the options/arguments sent to aquatone after processing. cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20 Returns: list: list of options/arguments, beginning with the name of the executable to run """ Path(self.output().path).mkdir(parents=True, exist_ok=True) command = [ tool_paths.get("aquatone"), "-scan-timeout", self.scan_timeout, "-threads", self.threads, "-silent", "-out", self.output().path, ] with self.input().open() as target_list: subprocess.run(command, stdin=target_list)
class SubjackScan(ExternalProgramTask): """ Use subjack to scan for potential subdomain takeovers. subjack commands are structured like the example below. subjack -w webtargets.tesla.txt -t 100 -timeout 30 -o subjack.tesla.txt -ssl An example of the corresponding luigi command is shown below. PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.subdomain_takeover SubjackScan --target-file tesla --top-ports 1000 --interface eth0 Args: threads: number of threads for parallel subjack command execution exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task top_ports: Scan top N most popular ports *--* Required by upstream Task ports: specifies the port(s) to be scanned *--* Required by upstream Task interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task """ threads = luigi.Parameter(default=defaults.get("threads", "")) def requires(self): """ SubjackScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. Naming convention for the output file is subjack.TARGET_FILE.txt. Returns: luigi.local_target.LocalTarget """ return luigi.LocalTarget( f"{self.results_dir}/subjack.{self.target_file}.txt") def program_args(self): """ Defines the options/arguments sent to subjack after processing. Returns: list: list of options/arguments, beginning with the name of the executable to run """ command = [ tool_paths.get("subjack"), "-w", self.input().path, "-t", self.threads, "-a", "-timeout", "30", "-o", self.output().path, "-v", "-ssl", "-c", tool_paths.get("subjack-fingerprints"), ] return command
def test_wordlist_exists(): if is_kali(): assert Path(defaults.get("gobuster-wordlist")).exists()
def test_threads_numeric(): assert defaults.get("threads").isnumeric()
def test_masscan_rate_numeric(): assert defaults.get("masscan-rate").isnumeric()
class CORScannerScan(ExternalProgramTask): """ Use CORScanner to scan for potential CORS misconfigurations. CORScanner commands are structured like the example below. python cors_scan.py -i webtargets.tesla.txt -t 100 An example of the corresponding luigi command is shown below. PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.corscanner CORScannerScan --target-file tesla --top-ports 1000 --interface eth0 Install: git clone https://github.com/chenjj/CORScanner.git cd CORScanner pip install -r requirements.txt pip install future Args: threads: number of threads for parallel subjack command execution exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task top_ports: Scan top N most popular ports *--* Required by upstream Task ports: specifies the port(s) to be scanned *--* Required by upstream Task interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task """ threads = luigi.Parameter(default=defaults.get("threads", "")) def requires(self): """ CORScannerScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. Naming convention for the output file is corscanner.TARGET_FILE.json. Returns: luigi.local_target.LocalTarget """ return luigi.LocalTarget( f"{self.results_dir}/corscanner.{self.target_file}.json") def program_args(self): """ Defines the options/arguments sent to tko-subs after processing. Returns: list: list of options/arguments, beginning with the name of the executable to run """ command = [ "python3", tool_paths.get("CORScanner"), "-i", self.input().path, "-t", self.threads, "-o", self.output().path, ] return command
class ThreadedNmapScan(luigi.Task): """ Run ``nmap`` against specific targets and ports gained from the ParseMasscanOutput Task. Install: ``nmap`` is already on your system if you're using kali. If you're not using kali, refer to your own distributions instructions for installing ``nmap``. Basic Example: .. code-block:: console nmap --open -sT -sC -T 4 -sV -Pn -p 43,25,21,53,22 -oA htb-targets-nmap-results/nmap.10.10.10.155-tcp 10.10.10.155 Luigi Example: .. code-block:: console PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.nmap ThreadedNmap --target-file htb-targets --top-ports 5000 Args: threads: number of threads for parallel nmap command execution rate: desired rate for transmitting packets (packets per second) *Required by upstream Task* interface: use the named raw network interface, such as "eth0" *Required by upstream Task* top_ports: Scan top N most popular ports *Required by upstream Task* ports: specifies the port(s) to be scanned *Required by upstream Task* target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task* results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task* """ threads = luigi.Parameter(default=defaults.get("threads", "")) def requires(self): """ ThreadedNmap depends on ParseMasscanOutput to run. TargetList expects target_file as a parameter. Masscan expects rate, target_file, interface, and either ports or top_ports as parameters. Returns: luigi.Task - ParseMasscanOutput """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, } return ParseMasscanOutput(**args) def output(self): """ Returns the target output for this task. Naming convention for the output folder is TARGET_FILE-nmap-results. The output folder will be populated with all of the output files generated by any nmap commands run. Because the nmap command uses -oA, there will be three files per target scanned: .xml, .nmap, .gnmap. Returns: luigi.local_target.LocalTarget """ results_subfolder = Path(self.results_dir) / "nmap-results" return luigi.LocalTarget(results_subfolder.resolve()) def run(self): """ Parses pickled target info dictionary and runs targeted nmap scans against only open ports. """ try: self.threads = abs(int(self.threads)) except TypeError: return logging.error( "The value supplied to --threads must be a non-negative integer." ) ip_dict = pickle.load(open(self.input().path, "rb")) nmap_command = [ # placeholders will be overwritten with appropriate info in loop below "nmap", "--open", "PLACEHOLDER-IDX-2", "-n", "-sC", "-T", "4", "-sV", "-Pn", "-p", "PLACEHOLDER-IDX-10", "-oA", ] commands = list() """ ip_dict structure { "IP_ADDRESS": {'udp': {"161", "5000", ... }, ... i.e. {protocol: set(ports) } } """ for target, protocol_dict in ip_dict.items(): for protocol, ports in protocol_dict.items(): tmp_cmd = nmap_command[:] tmp_cmd[2] = "-sT" if protocol == "tcp" else "-sU" # arg to -oA, will drop into subdir off curdir tmp_cmd[10] = ",".join(ports) tmp_cmd.append( str( Path(self.output().path) / f"nmap.{target}-{protocol}")) tmp_cmd.append(target) # target as final arg to nmap commands.append(tmp_cmd) # basically mkdir -p, won't error out if already there Path(self.output().path).mkdir(parents=True, exist_ok=True) with concurrent.futures.ThreadPoolExecutor( max_workers=self.threads) as executor: executor.map(subprocess.run, commands)
class GobusterScan(luigi.Task): """ Use gobuster to perform forced browsing. gobuster commands are structured like the example below. gobuster dir -q -e -k -t 20 -u www.tesla.com -w /usr/share/seclists/Discovery/Web-Content/common.txt -p http://127.0.0.1:8080 -o gobuster.tesla.txt -x php,html An example of the corresponding luigi command is shown below. PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.gobuster GobusterScan --target-file tesla --top-ports 1000 \ --interface eth0 --proxy http://127.0.0.1:8080 --extensions php,html \ --wordlist /usr/share/seclists/Discovery/Web-Content/common.txt --threads 20 Install: go get github.com/OJ/gobuster git clone https://github.com/epi052/recursive-gobuster.git Args: threads: number of threads for parallel gobuster command execution wordlist: wordlist used for forced browsing extensions: additional extensions to apply to each item in the wordlist recursive: whether or not to recursively gobust the target (may produce a LOT of traffic... quickly) exempt_list: Path to a file providing blacklisted subdomains, one per line. *--* Optional for upstream Task top_ports: Scan top N most popular ports *--* Required by upstream Task ports: specifies the port(s) to be scanned *--* Required by upstream Task interface: use the named raw network interface, such as "eth0" *--* Required by upstream Task rate: desired rate for transmitting packets (packets per second) *--* Required by upstream Task target_file: specifies the file on disk containing a list of ips or domains *--* Required by upstream Task results_dir: specifes the directory on disk to which all Task results are written *--* Required by upstream Task """ proxy = luigi.Parameter(default=defaults.get("proxy", "")) threads = luigi.Parameter(default=defaults.get("threads", "")) wordlist = luigi.Parameter(default=defaults.get("gobuster-wordlist", "")) extensions = luigi.Parameter(default=defaults.get("gobuster-extensions", "")) recursive = luigi.BoolParameter(default=False) def requires(self): """ GobusterScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. If recursion is disabled, the naming convention for the output file is gobuster.TARGET_FILE.txt Otherwise the output file is recursive-gobuster_TARGET_FILE.log Results are stored in their own directory: gobuster-TARGET_FILE-results Returns: luigi.local_target.LocalTarget """ return luigi.LocalTarget(f"{self.results_dir}/gobuster-{self.target_file}-results") def run(self): """ Defines the options/arguments sent to gobuster after processing. Returns: list: list of options/arguments, beginning with the name of the executable to run """ try: self.threads = abs(int(self.threads)) except TypeError: return logging.error("The value supplied to --threads must be a non-negative integer.") commands = list() with self.input().open() as f: for target in f: target = target.strip() try: if isinstance(ipaddress.ip_address(target), ipaddress.IPv6Address): # ipv6 target = f"[{target}]" except ValueError: # domain names raise ValueErrors, just assume we have a domain and keep on keepin on pass for url_scheme in ("https://", "http://"): if self.recursive: command = [ tool_paths.get("recursive-gobuster"), "-w", self.wordlist, f"{url_scheme}{target}", ] else: command = [ tool_paths.get("gobuster"), "dir", "-q", "-e", "-k", "-u", f"{url_scheme}{target}", "-w", self.wordlist, "-o", Path(self.output().path).joinpath( f"gobuster.{url_scheme.replace('//', '_').replace(':', '')}{target}.txt" ), ] if self.extensions: command.extend(["-x", self.extensions]) if self.proxy: command.extend(["-p", self.proxy]) commands.append(command) Path(self.output().path).mkdir(parents=True, exist_ok=True) if self.recursive: # workaround for recursive gobuster not accepting output directory cwd = Path().cwd() os.chdir(self.output().path) with ThreadPoolExecutor(max_workers=self.threads) as executor: executor.map(subprocess.run, commands) if self.recursive: os.chdir(str(cwd))
class AquatoneScan(luigi.Task): """ Screenshot all web targets and generate HTML report. Install: .. code-block:: console mkdir /tmp/aquatone wget -q https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O /tmp/aquatone/aquatone.zip unzip /tmp/aquatone/aquatone.zip -d /tmp/aquatone sudo mv /tmp/aquatone/aquatone /usr/local/bin/aquatone rm -rf /tmp/aquatone Basic Example: ``aquatone`` commands are structured like the example below. ``cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20`` Luigi Example: .. code-block:: python PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.aquatone AquatoneScan --target-file tesla --top-ports 1000 Args: threads: number of threads for parallel aquatone command execution scan_timeout: timeout in miliseconds for aquatone port scans exempt_list: Path to a file providing blacklisted subdomains, one per line. *Optional by upstream Task* top_ports: Scan top N most popular ports *Required by upstream Task* ports: specifies the port(s) to be scanned *Required by upstream Task* interface: use the named raw network interface, such as "eth0" *Required by upstream Task* rate: desired rate for transmitting packets (packets per second) *Required by upstream Task* target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task* results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task* """ threads = luigi.Parameter(default=defaults.get("threads", "")) scan_timeout = luigi.Parameter( default=defaults.get("aquatone-scan-timeout", "")) def requires(self): """ AquatoneScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. Naming convention for the output file is amass.TARGET_FILE.json. Returns: luigi.local_target.LocalTarget """ results_subfolder = Path(self.results_dir) / "aquatone-results" return luigi.LocalTarget(results_subfolder.resolve()) def run(self): """ Defines the options/arguments sent to aquatone after processing. cat webtargets.tesla.txt | /opt/aquatone -scan-timeout 900 -threads 20 Returns: list: list of options/arguments, beginning with the name of the executable to run """ Path(self.output().path).mkdir(parents=True, exist_ok=True) command = [ tool_paths.get("aquatone"), "-scan-timeout", self.scan_timeout, "-threads", self.threads, "-silent", "-out", self.output().path, ] with self.input().open() as target_list: subprocess.run(command, stdin=target_list)
def test_aquatone_scan_timeout_numeric(): assert defaults.get("aquatone-scan-timeout").isnumeric()
class MasscanScan(luigi.Task): """ Run ``masscan`` against a target specified via the TargetList Task. Note: When specified, ``--top_ports`` is processed and then ultimately passed to ``--ports``. Install: .. code-block:: console git clone https://github.com/robertdavidgraham/masscan /tmp/masscan make -s -j -C /tmp/masscan sudo mv /tmp/masscan/bin/masscan /usr/local/bin/masscan rm -rf /tmp/masscan Basic Example: .. code-block:: console masscan -v --open-only --banners --rate 1000 -e tun0 -oJ masscan.tesla.json --ports 80,443,22,21 -iL tesla.ips Luigi Example: .. code-block:: console PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.masscan Masscan --target-file tesla --ports 80,443,22,21 Args: rate: desired rate for transmitting packets (packets per second) interface: use the named raw network interface, such as "eth0" top_ports: Scan top N most popular ports ports: specifies the port(s) to be scanned target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task* results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task* exempt_list: Path to a file providing blacklisted subdomains, one per line. *Optional by upstream Task* """ rate = luigi.Parameter(default=defaults.get("masscan-rate", "")) interface = luigi.Parameter(default=defaults.get("masscan-iface", "")) top_ports = luigi.IntParameter( default=0) # IntParameter -> top_ports expected as int ports = luigi.Parameter(default="") def output(self): """ Returns the target output for this task. Naming convention for the output file is masscan.TARGET_FILE.json. Returns: luigi.local_target.LocalTarget """ results_subfolder = Path(self.results_dir) / "masscan-results" new_path = results_subfolder / "masscan.json" return luigi.LocalTarget(new_path.resolve()) def run(self): """ Defines the options/arguments sent to masscan after processing. Returns: list: list of options/arguments, beginning with the name of the executable to run """ if self.ports and self.top_ports: # can't have both logging.error( "Only --ports or --top-ports is permitted, not both.") exit(1) if not self.ports and not self.top_ports: # need at least one logging.error("Must specify either --top-ports or --ports.") exit(2) if self.top_ports < 0: # sanity check logging.error("--top-ports must be greater than 0") exit(3) if self.top_ports: # if --top-ports used, format the top_*_ports lists as strings and then into a proper masscan --ports option top_tcp_ports_str = ",".join( str(x) for x in top_tcp_ports[:self.top_ports]) top_udp_ports_str = ",".join( str(x) for x in top_udp_ports[:self.top_ports]) self.ports = f"{top_tcp_ports_str},U:{top_udp_ports_str}" self.top_ports = 0 target_list = yield TargetList(target_file=self.target_file, results_dir=self.results_dir) Path(self.output().path).parent.mkdir(parents=True, exist_ok=True) Path(self.output().path).parent.mkdir(parents=True, exist_ok=True) if target_list.path.endswith("domains"): yield ParseAmassOutput(target_file=self.target_file, exempt_list=self.exempt_list, results_dir=self.results_dir) command = [ "masscan", "-v", "--open", "--banners", "--rate", self.rate, "-e", self.interface, "-oJ", self.output().path, "--ports", self.ports, "-iL", ] if target_list.path.endswith("domains"): command.append( target_list.path.replace("domains", "ipv4_addresses")) else: command.append(target_list.path.replace("domains", "ip_addresses")) subprocess.run(command)
class SubjackScan(ExternalProgramTask): """ Use ``subjack`` to scan for potential subdomain takeovers. Install: .. code-block:: console go get github.com/haccer/subjack cd ~/go/src/github.com/haccer/subjack go build go install Basic Example: .. code-block:: console subjack -w webtargets.tesla.txt -t 100 -timeout 30 -o subjack.tesla.txt -ssl Luigi Example: .. code-block:: console PYTHONPATH=$(pwd) luigi --local-scheduler --module recon.web.subdomain_takeover SubjackScan --target-file tesla --top-ports 1000 --interface eth0 Args: threads: number of threads for parallel subjack command execution exempt_list: Path to a file providing blacklisted subdomains, one per line. *Optional by upstream Task* top_ports: Scan top N most popular ports *Required by upstream Task* ports: specifies the port(s) to be scanned *Required by upstream Task* interface: use the named raw network interface, such as "eth0" *Required by upstream Task* rate: desired rate for transmitting packets (packets per second) *Required by upstream Task* target_file: specifies the file on disk containing a list of ips or domains *Required by upstream Task* results_dir: specifes the directory on disk to which all Task results are written *Required by upstream Task* """ threads = luigi.Parameter(default=defaults.get("threads", "")) def requires(self): """ SubjackScan depends on GatherWebTargets to run. GatherWebTargets accepts exempt_list and expects rate, target_file, interface, and either ports or top_ports as parameters Returns: luigi.Task - GatherWebTargets """ args = { "results_dir": self.results_dir, "rate": self.rate, "target_file": self.target_file, "top_ports": self.top_ports, "interface": self.interface, "ports": self.ports, "exempt_list": self.exempt_list, } return GatherWebTargets(**args) def output(self): """ Returns the target output for this task. Naming convention for the output file is subjack.TARGET_FILE.txt. Returns: luigi.local_target.LocalTarget """ results_subfolder = Path(self.results_dir) / "subjack-results" new_path = results_subfolder / "subjack.txt" return luigi.LocalTarget(new_path.resolve()) def program_args(self): """ Defines the options/arguments sent to subjack after processing. Returns: list: list of options/arguments, beginning with the name of the executable to run """ Path(self.output().path).parent.mkdir(parents=True, exist_ok=True) command = [ tool_paths.get("subjack"), "-w", self.input().path, "-t", self.threads, "-a", "-timeout", "30", "-o", self.output().path, "-v", "-ssl", "-c", tool_paths.get("subjack-fingerprints"), ] return command