コード例 #1
0
ファイル: Subfinder.py プロジェクト: stjordanis/armory-1
class Module(ToolTemplate):
    name = "Subfinder"
    binary_name = "subfinder"

    def __init__(self, db):
        self.db = db
        self.BaseDomains = BaseDomainRepository(db, self.name)
        self.Domains = DomainRepository(db, self.name)
        self.IPs = IPRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()
        self.options.add_argument("-d",
                                  "--domain",
                                  help="Domain to run subfinder against.")
        self.options.add_argument(
            "-dL",
            "--domain_list",
            help="Read in a list of domains within the given file.",
        )
        self.options.add_argument(
            "-i",
            "--db_domains",
            help="Import the domains from the database.",
            action="store_true",
        )

        self.options.add_argument("--rescan",
                                  help="Overwrite files without asking",
                                  action="store_true")

    def get_targets(self, args):
        targets = []
        outpath = ""
        if args.output_path[0] == "/":
            output_path = os.path.join(
                self.base_config["PROJECT"]["base_path"], args.output_path[1:])
        else:
            output_path = os.path.join(
                self.base_config["PROJECT"]["base_path"], args.output_path)
        if not os.path.exists(output_path):
            os.makedirs(output_path)

        if args.domain:
            out_file = os.path.join(outpath,
                                    "{}.subfinder".format(args.domain))
            targets.append({
                "target": args.domain,
                "output": os.path.join(output_path, out_file)
            })

        if args.db_domains:
            if args.rescan:
                domains = self.BaseDomains.all(scope_type="passive")
            else:
                domains = self.BaseDomains.all(tool=self.name,
                                               scope_type="passive")
            for d in domains:
                out_file = os.path.join(outpath,
                                        "{}.subfinder".format(d.domain))
                targets.append({
                    "target": d.domain,
                    "output": os.path.join(output_path, out_file)
                })

        elif args.domain_list:
            domains = io.open(args.domain_list,
                              encoding="utf-8").read().split("\n")
            for d in domains:
                if d:
                    targets.append({
                        "target":
                        d,
                        "output":
                        os.path.join(output_path, "{}.subfinder".format(d)),
                    })

        return targets

    def build_cmd(self, args):
        if args.binary:
            cmd = "{} ".format(args.binary)
        else:
            cmd = "{} ".format(self.binary_name)
        cmd = "{} -o {} -d {}".format(cmd, "{output}", "{target}")
        return cmd

    def process_output(self, targets):
        for target in targets:
            try:
                with io.open(target["output"], encoding="utf-8") as fd:
                    for line in fd:
                        domain = line.strip()
                        if domain[0] == '.':
                            domain = domain[1:]
                        ips = get_domain_ip.run(domain)
                        ip_obj = None
                        _, dom = self.Domains.find_or_create(domain=domain)
                        if ips:
                            for ip in ips:
                                _, ip_obj = self.IPs.find_or_create(
                                    ip_address=ip)
                                if ip_obj:
                                    dom.ip_addresses.append(ip_obj)
                            dom.save()
            except FileNotFoundError:
                display_error("File doesn't exist for {}".format(
                    target["output"]))
        self.BaseDomains.commit()
        self.IPs.commit()

    def post_run(self, args):
        # Remove the temporary db file if it was created.
        if getattr(self, "db_domain_file", None):
            try:
                os.unlink(self.db_domain_file)
            except IOError as e:
                print("Failed to remove the Subfinder db temp file: '{}'.".
                      format(e))

    def __get_tempfile(self, domain=None, args=None):
        # Create a temporary file and place all of the current database domains within the file.
        from tempfile import NamedTemporaryFile

        with NamedTemporaryFile(delete=False) as fd:
            if domain:
                fd.write("{}\n".format(domain).encode("utf-8"))
            else:
                # Go through the database and grab the domains adding them to the file.
                if args.rescan:
                    domains = self.BaseDomains.all(passive_scope=True)
                else:
                    domains = self.BaseDomains.all(tool=self.name,
                                                   passive_scope=True)
                if domains:
                    for domain in domains:
                        fd.write("{}\n".format(domain.domain).encode("utf-8"))
                else:
                    return None
        return fd.name
コード例 #2
0
class Module(ModuleTemplate):
    """
    The Shodan module will either iterate through Shodan search results from net:<cidr>
    for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be
    added to the database, along with a dictionary object of the API results.

    """

    name = "ShodanImport"

    def __init__(self, db):
        self.db = db
        self.Port = PortRepository(db, self.name)
        self.IPAddress = IPRepository(db, self.name)
        self.ScopeCidr = ScopeCIDRRepository(db, self.name)
        self.Domain = DomainRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()

        self.options.add_argument("-k",
                                  "--api_key",
                                  help="API Key for accessing Shodan")
        self.options.add_argument(
            "-s", "--search", help="Custom search string (will use credits)")
        self.options.add_argument(
            "-i",
            "--import_db",
            help="Import scoped IPs from the database",
            action="store_true",
        )
        self.options.add_argument("--rescan",
                                  help="Rescan CIDRs already processed",
                                  action="store_true")
        self.options.add_argument("--fast",
                                  help="Use 'net' filter. (May use credits)",
                                  action="store_true")
        self.options.add_argument(
            "--cidr_only",
            help="Import only CIDRs from database (not individual IPs)",
            action="store_true",
        )

        self.options.add_argument("--target",
                                  "-t",
                                  help="Scan a specific CIDR/IP")

    def run(self, args):

        ranges = []
        cidrs = []
        ips = []
        search = []
        if not args.api_key:
            display_error("You must supply an API key to use shodan!")
            return

        if args.search:
            search = [args.search]

        if args.import_db:
            if args.rescan:
                if args.fast:
                    search += [
                        "net:{}".format(c.cidr) for c in self.ScopeCidr.all()
                    ]
                else:
                    cidrs += [c.cidr for c in self.ScopeCidr.all()]

                if not args.cidr_only:
                    ips += [
                        "{}".format(i.ip_address)
                        for i in self.IPAddress.all(scope_type="active")
                    ]
            else:
                if args.fast:
                    search += [
                        "net:{}".format(c.cidr)
                        for c in self.ScopeCidr.all(tool=self.name)
                    ]
                else:
                    cidrs += [
                        c.cidr for c in self.ScopeCidr.all(tool=self.name)
                    ]

                if not args.cidr_only:
                    ips += [
                        "{}".format(i.ip_address)
                        for i in self.IPAddress.all(scope_type="active",
                                                    tool=self.name)
                    ]
        if args.target:

            if '/' not in args.target:
                ips += [args.target]
            elif args.fast:
                cidrs += ["net:{}".format(args.target)]
            else:
                cidrs += [args.target]

        for c in cidrs:
            ranges += [str(i) for i in IPNetwork(c)]

        ranges += ips
        ranges += search

        display(
            "Doing a total of {} queries. Estimated time: {} days, {} hours, {} minutes and {} seconds."
            .format(len(ranges), int(len(ranges) / 24.0 / 60.0 / 60.0),
                    int(len(ranges) / 60.0 / 60.0) % 60,
                    int(len(ranges) / 60.0) % 60,
                    len(ranges) % 60))

        for c in cidrs:
            ranges = [str(i) for i in IPNetwork(c)]
            display(
                "Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
                .format(c, int(len(ranges) / 24.0 / 60.0 / 60.0),
                        int(len(ranges) / 60.0 / 60.0) % 60,
                        int(len(ranges) / 60.0) % 60,
                        len(ranges) % 60))
            for r in ranges:

                self.get_shodan(r, args)

            created, cd = self.ScopeCidr.find_or_create(cidr=c)
            if created:
                cd.delete()
            else:
                cd.set_tool(self.name)
            self.ScopeCidr.commit()
        display(
            "Processing {} IPs. Estimated time: {} days, {} hours, {} minutes and {} seconds."
            .format(len(ips), int(len(ranges) / 24.0 / 60.0 / 60.0),
                    int(len(ranges) / 60.0 / 60.0) % 60,
                    int(len(ranges) / 60.0) % 60,
                    len(ranges) % 60))
        for i in ips:
            self.get_shodan(i, args)

            created, ip = self.IPAddress.find_or_create(ip_address=i)
            if created:
                ip.delete()
            else:
                ip.set_tool(self.name)
            self.IPAddress.commit()

        for s in search:
            self.get_shodan(s, args)

            if s[:4] == "net:":
                created, cd = self.ScopeCidr.find_or_create(cidr=s[4:])
                if created:
                    cd.delete()
                else:
                    cd.set_tool(self.name)
                self.ScopeCidr.commit()

    def get_shodan(self, r, args):

        api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
        api_search_url = (
            "https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
        time.sleep(1)
        if ":" in r:
            display("Doing Shodan search: {}".format(r))
            try:
                results = json.loads(
                    requests.get(api_search_url.format(args.api_key, r,
                                                       1)).text)
                if results.get(
                        "error") and "request timed out" in results["error"]:
                    display_warning(
                        "Timeout occurred on Shodan's side.. trying again in 5 seconds."
                    )
                    results = json.loads(
                        requests.get(api_search_url.format(args.api_key, r,
                                                           1)).text)
            except Exception as e:
                display_error("Something went wrong: {}".format(e))
                next

            total = len(results["matches"])
            matches = []
            i = 1
            # pdb.set_trace()
            while total > 0:
                display("Adding {} results from page {}".format(total, i))
                matches += results["matches"]
                i += 1
                try:
                    time.sleep(1)
                    results = json.loads(
                        requests.get(api_search_url.format(args.api_key, r,
                                                           i)).text)
                    if (results.get("error") and "request timed out"
                            in results["error"]  # noqa: W503
                        ):
                        display_warning(
                            "Timeout occurred on Shodan's side.. trying again in 5 seconds."
                        )
                        results = json.loads(
                            requests.get(
                                api_search_url.format(args.api_key, r,
                                                      1)).text)

                    total = len(results["matches"])

                except Exception as e:
                    display_error("Something went wrong: {}".format(e))
                    total = 0
                    pdb.set_trace()
            domains = []

            for res in matches:
                ip_str = res["ip_str"]
                port_str = res["port"]
                transport = res["transport"]

                display("Processing IP: {} Port: {}/{}".format(
                    ip_str, port_str, transport))

                created, IP = self.IPAddress.find_or_create(ip_address=ip_str)
                IP.meta["shodan_data"] = results

                created, port = self.Port.find_or_create(ip_address=IP,
                                                         port_number=port_str,
                                                         proto=transport)
                if created:
                    svc = ""

                    if res.get("ssl", False):
                        svc = "https"
                    elif res.get("http", False):
                        svc = "http"

                    else:
                        svc = ""

                    port.service_name = svc
                port.status = "open"
                port.meta["shodan_data"] = res
                port.save()

                if res.get("ssl", {}).get('cert', {}).get('extensions'):
                    for d in res['ssl']['cert']['extensions']:
                        if d['name'] == 'subjectAltName':
                            domains += get_domains_from_data(d['name'])

                if res.get("ssl", {}).get('cert', {}).get(
                        'subject', {}
                ).get('CN') and '*' not in res['ssl']['cert']['subject']['CN']:
                    domains.append(res['ssl']['cert']['subject']['CN'])

                if res.get('hostnames'):
                    domains += res['hostnames']

            for d in list(set(domains)):
                display("Adding discovered domain {}".format(only_valid(d)))
                created, domain = self.Domain.find_or_create(
                    domain=only_valid(d))

        else:
            display("Searching for {}".format(r))
            try:
                results = json.loads(
                    requests.get(api_host_url.format(r, args.api_key)).text)
            except Exception as e:
                display_error("Something went wrong: {}".format(e))
                next
            # pdb.set_trace()
            if results.get("data", False):

                display("{} results found for: {}".format(
                    len(results["data"]), r))
                domains = []
                for res in results["data"]:
                    ip_str = res["ip_str"]
                    port_str = res["port"]
                    transport = res["transport"]
                    display("Processing IP: {} Port: {}/{}".format(
                        ip_str, port_str, transport))
                    created, IP = self.IPAddress.find_or_create(
                        ip_address=ip_str)
                    IP.meta["shodan_data"] = results

                    created, port = self.Port.find_or_create(
                        ip_address=IP, port_number=port_str, proto=transport)

                    if created:
                        svc = ""

                        if res.get("ssl", False):
                            svc = "https"
                        elif res.get("http", False):
                            svc = "http"

                        else:
                            svc = ""

                        port.service_name = svc
                    port.status = "open"
                    port.meta["shodan_data"] = res
                    port.save()

                    if res.get("ssl", {}).get('cert', {}).get('extensions'):
                        for d in res['ssl']['cert']['extensions']:
                            if d['name'] == 'subjectAltName':

                                domains += get_domains_from_data(d['data'])
                                display(
                                    "Domains discovered in subjectAltName: {}".
                                    format(", ".join(
                                        get_domains_from_data(d['data']))))

                    if res.get("ssl", {}).get('cert', {}).get(
                            'subject', {}).get('CN') and '*' not in res['ssl'][
                                'cert']['subject']['CN']:
                        domains.append(res['ssl']['cert']['subject']['CN'])

                    if res.get('hostnames'):
                        domains += res['hostnames']

                for d in list(set(domains)):
                    display("Adding discovered domain {}".format(d))
                    created, domain = self.Domain.find_or_create(domain=d)
コード例 #3
0
ファイル: Nmap.py プロジェクト: zz0505/armory
class Module(ToolTemplate):
    """
    Module for running nmap. Make sure to pass all nmap-specific arguments at the end, after --tool_args

    """

    name = "Nmap"
    binary_name = "nmap"

    def __init__(self, db):
        self.db = db
        self.BaseDomain = BaseDomainRepository(db, self.name)
        self.Domain = DomainRepository(db, self.name)
        self.IPAddress = IPRepository(db, self.name)
        self.Port = PortRepository(db, self.name)

        self.Vulnerability = VulnRepository(db, self.name)
        self.CVE = CVERepository(db, self.name)
        self.ScopeCIDR = ScopeCIDRRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()
        self.options.add_argument(
            "--hosts",
            help=
            "Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS",
            nargs="+",
        )
        self.options.add_argument("--hosts_file", help="File containing hosts")
        self.options.add_argument(
            "-i",
            "--hosts_database",
            help="Use unscanned hosts from the database",
            action="store_true",
        )
        self.options.add_argument("--rescan",
                                  help="Overwrite files without asking",
                                  action="store_true")
        self.options.add_argument(
            "--filename",
            help="Output filename. By default will use the current timestamp.",
        )
        self.options.add_argument(
            "--ssl_cert_mode",
            help=
            "Scan only SSL enabled hosts to collect SSL certs (and domain names)",
            action="store_true",
        )
        self.options.add_argument(
            "--filter_ports",
            help=
            "Comma separated list of protoPort to filter out of results. Useful if firewall returns specific ports open on every host. Ex: t80,u5060"
        )

        self.options.set_defaults(timeout=None)
        self.options.add_argument("--import_file",
                                  help="Import results from an Nmap XML file.")

    def get_targets(self, args):

        self.args = args
        if args.import_file:
            args.no_binary = True
            return [{"target": "", "output": args.import_file}]

        targets = []

        if args.hosts:

            if type(args.hosts) == list:
                for h in args.hosts:
                    if check_if_ip(h):
                        targets.append(h)
                    else:
                        created, domain = self.Domain.find_or_create(domain=h)
                        targets += [i.ip_address for i in domain.ip_addresses]

            else:
                if check_if_ip(h):
                    targets.append(h)
                else:
                    created, domain = self.Domain.find_or_create(domain=h)
                    targets += [i.ip_address for i in domain.ip_addresses]

        if args.hosts_database:
            if args.rescan:
                targets += [
                    h.ip_address
                    for h in self.IPAddress.all(scope_type="active")
                ]
                targets += [h.cidr for h in self.ScopeCIDR.all()]
            else:
                targets += [
                    h.ip_address
                    for h in self.IPAddress.all(tool=self.name,
                                                scope_type="active")
                ]
                targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)]

        if args.hosts_file:
            for h in [
                    l for l in open(args.hosts_file).read().split("\n") if l
            ]:
                if check_if_ip(h):
                    targets.append(h)
                else:
                    created, domain = self.Domain.find_or_create(domain=h)
                    targets += [i.ip_address for i in domain.ip_addresses]

        data = []
        if args.ssl_cert_mode:
            ports = self.Port.all(service_name='https')

            data = list(set([i.ip_address.ip_address for i in ports]))

            port_numbers = list(set([str(i.port_number) for i in ports]))
            args.tool_args += " -sV -p {} --script ssl-cert ".format(
                ','.join(port_numbers))

        else:

            # Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs
            for t in targets:
                ips = [str(i) for i in list(IPNetwork(t))]
                data += ips

        _, file_name = tempfile.mkstemp()
        open(file_name, "w").write("\n".join(list(set(data))))

        if args.output_path[0] == "/":
            self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
                                     args.output_path[1:])
        else:
            self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
                                     args.output_path)

        if not os.path.exists(self.path):
            os.makedirs(self.path)

        if args.filename:
            output_path = os.path.join(self.path, args.filename)
        else:
            output_path = os.path.join(
                self.path,
                "nmap-scan-%s.xml" %
                datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"),
            )

        return [{"target": file_name, "output": output_path}]

    def build_cmd(self, args):

        command = "sudo " + self.binary + " -oX {output} -iL {target} "

        if args.tool_args:
            command += args.tool_args

        return command

    def process_output(self, cmds):

        self.import_nmap(cmds[0]["output"])
        if cmds[0]["target"]:
            os.unlink(cmds[0]["target"])

    def parseHeaders(self, httpHeaders):
        bsHeaders = [
            "Pragma",
            "Expires",
            "Date",
            "Transfer-Encoding",
            "Connection",
            "X-Content-Type-Options",
            "Cache-Control",
            "X-Frame-Options",
            "Content-Type",
            "Content-Length",
            "(Request type",
        ]
        keepHeaders = {}
        for i in range(0, len(httpHeaders)):
            if httpHeaders[i].strip() != "" and httpHeaders[i].split(
                    ":")[0].strip() not in " ".join(bsHeaders):
                hName = httpHeaders[i].split(":")[0].strip()
                hValue = "".join(httpHeaders[i].split(":")[1:]).strip()
                keepHeaders[hName] = hValue

        if keepHeaders == {}:
            keepHeaders = ""

        return keepHeaders

    def import_nmap(
            self,
            filename):  # domains={}, ips={}, rejects=[] == temp while no db
        nFile = filename
        # pdb.set_trace()
        try:
            tree = ET.parse(nFile)
            root = tree.getroot()
            hosts = root.findall("host")

        except Exception as e:
            print("Error: {}" % e)
            print(nFile + " doesn't exist somehow...skipping")
            return

        for host in hosts:
            hostIP = host.find("address").get("addr")

            created, ip = self.IPAddress.find_or_create(ip_address=hostIP)

            for hostname in host.findall("hostnames/hostname"):
                hostname = hostname.get("name")
                hostname = hostname.lower().replace("www.", "")

                # reHostname = re.search(
                #     r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname
                # )  # attempt to not get PTR record
                # if not reHostname:

                created, domain = self.Domain.find_or_create(domain=hostname)
                if ip not in domain.ip_addresses:
                    domain.ip_addresses.append(ip)
                    domain.save()

            for port in host.findall("ports/port"):

                if port.find("state").get("state"):
                    portState = port.find("state").get("state")
                    hostPort = port.get("portid")
                    portProto = port.get("protocol")
                    # pdb.set_trace()
                    if not self.args.filter_ports or int(hostPort) not in [
                            int(p) for p in self.args.filter_ports.split(',')
                    ]:
                        created, db_port = self.Port.find_or_create(
                            port_number=hostPort,
                            status=portState,
                            proto=portProto,
                            ip_address=ip,
                        )

                        if port.find("service") is not None:
                            portName = port.find("service").get("name")
                            if portName == "http" and hostPort == "443":
                                portName = "https"
                        else:
                            portName = "Unknown"

                        if created:
                            db_port.service_name = portName
                        info = db_port.info
                        if not info:
                            info = {}

                        for script in port.findall(
                                "script"):  # just getting commonName from cert
                            if script.get("id") == "ssl-cert":
                                db_port.cert = script.get("output")
                                cert_domains = self.get_domains_from_cert(
                                    script.get("output"))

                                for hostname in cert_domains:
                                    hostname = hostname.lower().replace(
                                        "www.", "")
                                    created, domain = self.Domain.find_or_create(
                                        domain=hostname)
                                    if created:
                                        print("New domain found: %s" %
                                              hostname)

                            elif script.get("id") == "vulners":
                                print("Gathering vuln info for {} : {}/{}\n".
                                      format(hostIP, portProto, hostPort))
                                self.parseVulners(script.get("output"),
                                                  db_port)

                            elif script.get("id") == "banner":
                                info["banner"] = script.get("output")

                            elif script.get("id") == "http-headers":

                                httpHeaders = script.get("output")
                                httpHeaders = httpHeaders.strip().split("\n")
                                keepHeaders = self.parseHeaders(httpHeaders)
                                info["http-headers"] = keepHeaders

                            elif script.get("id") == "http-auth":
                                info["http-auth"] = script.get("output")

                            elif script.get("id") == "http-title":
                                info["http-title"] = script.get("output")

                        db_port.info = info
                        db_port.save()

            self.IPAddress.commit()

    def parseVulners(self, scriptOutput, db_port):
        urls = re.findall(r"(https://vulners.com/cve/CVE-\d*-\d*)",
                          scriptOutput)
        for url in urls:
            vuln_refs = []
            exploitable = False
            cve = url.split("/cve/")[1]
            vulners = requests.get("https://vulners.com/cve/%s" % cve).text
            exploitdb = re.findall(
                r"https://www.exploit-db.com/exploits/\d{,7}", vulners)
            for edb in exploitdb:
                exploitable = True

                if edb.split("/exploits/")[1] not in vuln_refs:
                    vuln_refs.append(edb.split("/exploits/")[1])

            if not self.CVE.find(name=cve):
                # print "Gathering CVE info for", cve
                try:
                    res = json.loads(
                        requests.get("http://cve.circl.lu/api/cve/%s" %
                                     cve).text)
                    cveDescription = res["summary"]
                    cvss = float(res["cvss"])
                    findingName = res["oval"][0]["title"]
                    if int(cvss) <= 3:
                        severity = 1

                    elif (int(cvss) / 2) == 5:
                        severity = 4

                    else:
                        severity = int(cvss) / 2

                    if not self.Vulnerability.find(name=findingName):
                        # print "Creating", findingName
                        created, db_vuln = self.Vulnerability.find_or_create(
                            name=findingName,
                            severity=severity,
                            description=cveDescription,
                        )
                        db_vuln.ports.append(db_port)
                        db_vuln.exploitable = exploitable
                        if vuln_refs:
                            db_vuln.exploit_reference = {"edb-id": vuln_refs}
                            db_vuln.save()

                    else:
                        # print "modifying",findingName
                        db_vuln = self.Vulnerability.find(name=findingName)
                        db_vuln.ports.append(db_port)
                        db_vuln.exploitable = exploitable

                        if vuln_refs:
                            db_vuln.exploitable = exploitable
                            if db_vuln.exploit_reference is not None:
                                if "edb-id" in db_vuln.exploit_reference:
                                    for ref in vuln_refs:
                                        if (ref not in db_vuln.
                                                exploit_reference["edb-id"]):
                                            db_vuln.exploit_reference[
                                                "edb-id"].append(ref)

                                else:
                                    db_vuln.exploit_reference[
                                        "edb-id"] = vuln_refs
                            else:
                                db_vuln.exploit_reference = {
                                    "edb-id": vuln_refs
                                }

                        db_vuln.save()

                    if not self.CVE.find(name=cve):
                        created, db_cve = self.CVE.find_or_create(
                            name=cve,
                            description=cveDescription,
                            temporal_score=cvss)
                        db_cve.vulnerabilities.append(db_vuln)
                        db_cve.save()

                    else:
                        db_cve = self.CVE.find(name=cve)
                        db_cve.vulnerabilities.append(db_vuln)
                        db_cve.save()

                    self.Vulnerability.commit()
                    self.CVE.commit()
                except Exception:
                    print(
                        "something went wrong with the vuln/cve info gathering"
                    )
                    if vulners:
                        print(
                            "Vulners report was found but no exploit-db was discovered"
                        )
                        # "Affected vulners items"
                        # print vulners
                    print("Affected CVE")
                    print(cve)
                    pass

            else:
                db_cve = self.CVE.find(name=cve)
                for db_vulns in db_cve.vulnerabilities:
                    if db_port not in db_vulns.ports:
                        db_vulns.ports.append(db_port)
        return

    def get_domains_from_cert(self, cert):
        # Shamelessly lifted regex from stack overflow
        regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}"

        domains = list(
            set([d for d in re.findall(regex, cert) if "*" not in d]))

        return domains
コード例 #4
0
ファイル: Nessus.py プロジェクト: stjordanis/armory-1
class Module(ModuleTemplate):

    name = "Nessus"

    def __init__(self, db):
        self.db = db
        self.BaseDomain = BaseDomainRepository(db, self.name)
        self.Domain = DomainRepository(db, self.name)
        self.IPAddress = IPRepository(db, self.name)
        self.Port = PortRepository(db, self.name)
        self.Vulnerability = VulnRepository(db, self.name)
        self.CVE = CVERepository(db, self.name)
        self.ScopeCIDR = ScopeCIDRRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()
        self.options.add_argument(
            "--import_file",
            help=
            "Import separated Nessus files separated by a space. DO NOT USE QUOTES OR COMMAS",
            nargs="+",
        )
        self.options.add_argument(
            "--launch",
            help=
            "Launch Nessus scan using Actively scoped IPs and domains in the database",
            action="store_true",
        )
        self.options.add_argument("--job_name",
                                  help="Job name inside Nessus",
                                  default="Armory Job")
        self.options.add_argument("--username", help="Nessus Username")
        self.options.add_argument("--password", help="Nessus Password")
        self.options.add_argument(
            "--host",
            help="Hostname:Port of Nessus web interface (ie localhost:8835")
        self.options.add_argument("--uuid",
                                  help="UUID of Nessus Policy to run")
        self.options.add_argument("--policy_id", help="Policy ID to use")
        self.options.add_argument("--folder_id",
                                  help="ID for folder to store job in")
        self.options.add_argument(
            "--download",
            help="Download Nessus job from server and import",
            action="store_true",
        )
        self.options.add_argument("--job_id",
                                  help="Job ID to download and import")
        self.options.add_argument(
            "--output_path",
            help="Path to store downloaded file (Default: Nessus)",
            default=self.name,
        )
        self.options.add_argument(
            "--disable_mitre",
            help="Disable mitre CVE data gathering.",
            action="store_true",
        )

    def run(self, args):
        if args.import_file:
            for nFile in args.import_file:
                self.process_data(nFile, args)
        elif args.launch:
            if (not args.username  # noqa: W503
                    and not args.password  # noqa: W503
                    and not args.host  # noqa: W503
                    and not args.uuid  # noqa: W503
                    and not args.policy_id  # noqa: W503
                    and not args.folder_id  # noqa: W503
                ):
                display_error(
                    "You must supply a username, password, and host to launch a Nessus job"
                )

            else:
                n = NessusRequest(
                    args.username,
                    args.password,
                    args.host,
                    uuid=args.uuid,
                    policy_id=args.policy_id,
                    folder_id=args.folder_id,
                )

                ips = [
                    ip.ip_address
                    for ip in self.IPAddress.all(scope_type="active",
                                                 tool=self.name)
                ]
                cidrs = [
                    cidr.cidr for cidr in self.ScopeCIDR.all(tool=self.name)
                ]
                domains = [
                    domain.domain
                    for domain in self.Domain.all(scope_type="active",
                                                  tool=self.name)
                ]
                targets = ", ".join(merge_ranges(ips + cidrs) + domains)

                res = n.launch_job(targets, args.job_name)
                display("New Nessus job launched with ID {}".format(res))
                display(
                    "Remember this number! You'll need it to download the job once it is done."
                )

        elif args.download:
            if (not args.username  # noqa: W503
                    and not args.password  # noqa: W503
                    and not args.host  # noqa: W503
                    and not args.job_id  # noqa: W503
                ):
                display_error(
                    "You must supply host, username, password and job_id to download a report to import"
                )

            else:
                n = NessusRequest(
                    args.username,
                    args.password,
                    args.host,
                )

                if args.output_path[0] == "/":
                    output_path = os.path.join(
                        self.base_config["PROJECT"]["base_path"],
                        args.output_path[1:])
                else:
                    output_path = os.path.join(
                        self.base_config["PROJECT"]["base_path"],
                        args.output_path)

                if not os.path.exists(output_path):
                    os.makedirs(output_path)
                output_path = os.path.join(
                    output_path,
                    "Nessus-export-{}.nessus".format(int(time.time())))
                n.export_file(args.job_id, output_path)

                self.process_data(output_path, args)

    def nessCheckPlugin(self, tag):
        nessPlugins = [
            "10759",
            "77026",
            "20089",
            "56984",
            "71049",
            "70658",
            "40984",
            "11411",
        ]

        pluginID = tag.get("pluginID")

        if pluginID in nessPlugins:
            # print pluginID + " is in the list"
            if pluginID == "10759":
                if tag.find("plugin_output") is not None:
                    return (
                        tag.find("plugin_output").text.split(
                            "\n\n")[3].strip()
                    )  # returns IP for Web Server HTTP Header INternal IP disclosure
                else:
                    return ""

            if pluginID == "77026":
                if tag.find("plugin_output") is not None:
                    return (
                        tag.find("plugin_output").text.split(
                            "\n\n")[3].strip()
                    )  # Microsoft Exchange Client Access Server Information Disclosure (IP addy)
                else:
                    return ""

            if pluginID == "71049" or pluginID == "70658":
                output = ""
                if tag.find("plugin_output") is not None:
                    tmp = tag.find("plugin_output").text.split(":")[
                        1]  # SSH Weak MAC & CBC Algorithms Enabled
                    # print "#"*5
                    tmp = tmp.split("\n\n")[1].replace("  ", "")
                    # print "#"*5
                    output = tmp.split("\n")
                    # print ", ".join(output)
                return ", ".join(output)

            if pluginID == "56984":
                if tag.find("plugin_output") is not None:
                    tmp = (tag.find("plugin_output").text.split(
                        "This port supports ")[1].strip()
                           )  # SSL / TLS Versions Supported
                    tmp = tmp.split("/")
                    bad = []
                    for i in tmp:
                        # print i
                        if "SSLv" in i:
                            bad.append(i)
                        elif "TLSv1.0" in i:
                            bad.append(i)
                    if bad != []:
                        return ", ".join(bad).rstrip(".")
                    else:
                        return ""
                else:
                    return ""

            if pluginID == "40984":  # browsable web dirs
                if tag.find("plugin_output") is not None:
                    tmp = (tag.find("plugin_output").text.split(
                        "The following directories are browsable :")
                           [1].strip())
                    directories = tmp.split("\n")

                    return "\n".join(directories)

            if pluginID == "11411":  # Backup Files Disclosure
                if tag.find("plugin_output") is not None:
                    urls = []

                    tmp = (tag.find("plugin_output").text.split(
                        "It is possible to read the following backup file")
                           [1].strip())
                    tmpUrls = tmp.split("\n")
                    for url in tmpUrls:
                        if "URL" in url:
                            urls.append(url.split(":")[1].lstrip())
                    if urls:
                        return "\n".join(urls)
                    else:
                        return ""

            if pluginID == "20089":  # F5 cookie
                if tag.find("plugin_output") is not None:
                    f5Output = []
                    cookieVal = []
                    output = tag.find("plugin_output").text.strip().split("\n")
                    for line in output:
                        # print line
                        line = line.split(":")
                        for i, item in enumerate(line):
                            item = item.strip()
                            if "Cookie" in item:
                                line.pop(i)  # Pop to remove the first?
                                tmp = line.pop(i)
                                tmp.strip()
                                cookieVal.append(tmp)
                            else:
                                item = "".join(item)
                                f5Output.append(item)

                    f5Output = " : ".join(f5Output)
                    f5Output = f5Output.replace(" :  : ", ", ")
                    f5Output += " [" + ", ".join(cookieVal) + "]"
                    c = 0
                    tmpF5Output = f5Output.split()
                    for i, letter in enumerate(tmpF5Output):
                        if letter == ":":
                            c += 1
                            if (c % 2) == 0:
                                tmpF5Output[i] = " "
                                return "".join(tmpF5Output).replace("[", " [")
                else:

                    return ""
        else:

            return False

    def getVulns(self, ip, ReportHost):
        """Gets vulns and associated services"""
        for tag in ReportHost.iter("ReportItem"):
            exploitable = False
            cves = []
            vuln_refs = {}
            proto = tag.get("protocol")
            port = tag.get("port")
            svc_name = tag.get("svc_name").replace("?", "")
            plugin_output = []

            tmpPort = proto + "/" + port
            if tmpPort.lower() == "tcp/443":
                portName = "https"
            elif tmpPort.lower() == "tcp/80":
                portName = "http"
            elif svc_name == "www":
                plugin_name = tag.get("pluginName")
                if "tls" in plugin_name.lower() or "ssl" in plugin_name.lower(
                ):
                    portName = "https"
                else:
                    portName = "http"
            else:
                portName = svc_name

            created, db_port = self.Port.find_or_create(port_number=port,
                                                        status="open",
                                                        proto=proto,
                                                        ip_address_id=ip.id)

            if db_port.service_name == "http":
                if portName == "https":
                    db_port.service_name = portName
            elif db_port.service_name == "https":
                pass
            else:
                db_port.service_name = portName
            db_port.save()

            if tag.get("pluginID") == "56984":
                severity = 1
            elif tag.get("pluginID") == "11411":
                severity = 3
            else:
                severity = int(tag.get("severity"))

            findingName = tag.get("pluginName")
            description = tag.find("description").text

            if tag.find(
                    "solution") is not None and tag.find("solution") != "n/a":
                solution = tag.find("solution").text
            else:
                solution = "No Remediation From Nessus"

            nessCheck = self.nessCheckPlugin(tag)

            if nessCheck:
                if not db_port.info:
                    db_port.info = {findingName: nessCheck}
                else:
                    db_port.info[findingName] = nessCheck

                db_port.save()

            if tag.find("exploit_available") is not None:
                exploitable = True

            metasploits = tag.findall("metasploit_name")
            if metasploits:
                vuln_refs["metasploit"] = []
                for tmp in metasploits:
                    vuln_refs["metasploit"].append(tmp.text)

            edb_id = tag.findall("edb-id")
            if edb_id:
                vuln_refs["edb-id"] = []
                for tmp in edb_id:
                    vuln_refs["edb-id"].append(tmp.text)

            tmpcves = tag.findall("cve")
            for c in tmpcves:
                if c.text not in cves:
                    cves.append(c.text)

            cwe_ids = [c.text for c in tag.findall("cwe")]
            references = [c.text for c in tag.findall("see_also")]

            if not self.Vulnerability.find(name=findingName):
                created, db_vuln = self.Vulnerability.find_or_create(
                    name=findingName,
                    severity=severity,
                    description=description,
                    remediation=solution,
                )
                db_vuln.ports.append(db_port)
                db_vuln.exploitable = exploitable
                if exploitable:
                    display_new("exploit avalable for " + findingName)

                if vuln_refs:
                    db_vuln.exploit_reference = vuln_refs

            else:
                db_vuln = self.Vulnerability.find(name=findingName)
                db_vuln.ports.append(db_port)
                db_vuln.exploitable = exploitable
                if vuln_refs:
                    if db_vuln.exploit_reference is not None:
                        for key in vuln_refs.keys():
                            if key not in db_vuln.exploit_reference.keys():
                                db_vuln.exploit_reference[key] = vuln_refs[key]
                            else:
                                for ref in vuln_refs[key]:
                                    if ref not in db_vuln.exploit_reference[
                                            key]:
                                        db_vuln.exploit_reference[key].append(
                                            ref)
                    else:
                        db_vuln.exploit_reference = vuln_refs
            db_vuln.meta['CWEs'] = cwe_ids
            db_vuln.meta['Refs'] = references

            if tag.find("plugin_output") is not None:

                plugin_output = tag.find("plugin_output").text
                if not db_vuln.meta.get('plugin_output', False):
                    db_vuln.meta['plugin_output'] = {}
                if not db_vuln.meta['plugin_output'].get(ip.ip_address, False):
                    db_vuln.meta['plugin_output'][ip.ip_address] = {}

                if not db_vuln.meta['plugin_output'][ip.ip_address].get(
                        port, False):
                    db_vuln.meta['plugin_output'][ip.ip_address][port] = []

                if plugin_output not in db_vuln.meta['plugin_output'][
                        ip.ip_address][port]:

                    db_vuln.meta['plugin_output'][ip.ip_address][port].append(
                        plugin_output)

            if not self.args.disable_mitre:
                for cve in cves:
                    if not self.CVE.find(name=cve):
                        try:
                            url = 'https://nvd.nist.gov/vuln/detail//{}'
                            res = requests.get(url.format(cve)).text

                            cveDescription = res.split(
                                '<p data-testid="vuln-description">')[1].split(
                                    '</p>')[0]
                            if 'vuln-cvssv3-base-score' in res:
                                cvss = float(
                                    res.split(
                                        '<span data-testid="vuln-cvssv3-base-score">'
                                    )[1].split('</span>')[0].strip())
                            else:
                                cvss = float(
                                    res.split(
                                        '<span data-testid="vuln-cvssv2-base-score">'
                                    )[1].split('</span>')[0].strip())

                            cveDescription = res["summary"]
                            cvss = float(res["cvss"])

                        except Exception:
                            cveDescription = None
                            cvss = None

                        if not self.CVE.find(name=cve):
                            created, db_cve = self.CVE.find_or_create(
                                name=cve,
                                description=cveDescription,
                                temporal_score=cvss,
                            )
                            db_cve.vulnerabilities.append(db_vuln)
                        else:
                            db_cve = self.CVE.find(name=cve)
                            if (db_cve.description is None and
                                    cveDescription is not None  # noqa: W503
                                ):
                                db_cve.description = cveDescription
                            if db_cve.temporal_score is None and cvss is not None:
                                db_cve.temporal_score = cvss
                            db_cve.vulnerabilities.append(db_vuln)

    def process_data(self, nFile, args):
        display("Reading " + nFile)
        tree = ET.parse(nFile)
        root = tree.getroot()
        self.args = args
        for ReportHost in root.iter("ReportHost"):
            os = []
            hostname = ""
            hostIP = ""
            for HostProperties in ReportHost.iter("HostProperties"):
                for tag in HostProperties:
                    if tag.get("name") == "host-ip":
                        hostIP = tag.text

                    if tag.get("name") == "host-fqdn":
                        hostname = tag.text.lower()
                        hostname = hostname.replace("www.", "")

                    if tag.get("name") == "operating-system":
                        os = tag.text.split("\n")

            if hostIP:  # apparently nessus doesn't always have an IP to work with...

                if hostname:
                    display("Gathering Nessus info for {} ( {} )".format(
                        hostIP, hostname))
                else:
                    display("Gathering Nessus info for {}".format(hostIP))

                created, ip = self.IPAddress.find_or_create(ip_address=hostIP)

                if hostname:
                    created, domain = self.Domain.find_or_create(
                        domain=hostname)

                    if ip not in domain.ip_addresses:
                        ip.save()
                        domain.ip_addresses.append(ip)
                        domain.save()

                if os:
                    for o in os:
                        if not ip.OS:
                            ip.OS = o
                        else:
                            if o not in ip.OS.split(" OR "):
                                ip.OS += " OR " + o

                self.getVulns(ip, ReportHost)
                self.IPAddress.commit()

        return
コード例 #5
0
ファイル: JexBoss.py プロジェクト: stjordanis/armory-1
class Module(ToolTemplate):

    name = "JexBoss"
    binary_name = "jexboss.py"

    def __init__(self, db):
        self.db = db
        self.IPAddress = IPRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()

        self.options.add_argument(
            "-i",
            "--import_database",
            help="Import URLs from the database",
            action="store_true",
        )
        self.options.add_argument("-f",
                                  "--import_file",
                                  help="Import URLs from file")

    def get_targets(self, args):

        timestamp = str(int(time()))
        targets = []
        if args.import_file:
            targets += [
                t for t in open(args.import_file).read().split("\n") if t
            ]

        if args.import_database:
            if args.rescan:
                targets += get_urls.run(self.db, scope_type="active")
            else:
                targets += get_urls.run(self.db,
                                        scope_type="active",
                                        tool=self.name)

        if targets:
            if args.output_path[0] == "/":
                self.path = os.path.join(
                    self.base_config["PROJECT"]["base_path"],
                    args.output_path[1:],
                    timestamp,
                )
            else:
                self.path = os.path.join(
                    self.base_config["PROJECT"]["base_path"],
                    args.output_path,
                    timestamp,
                )
            if not os.path.exists(self.path):
                os.makedirs(self.path)

            res = []
            i = 0
            if args.group_size == 0:
                args.group_size = len(targets)

            for url_chunk in self.chunks(targets, args.group_size):
                i += 1

                _, file_name = tempfile.mkstemp()
                open(file_name, "w").write("\n".join(url_chunk))

                res.append({
                    "target": file_name,
                    "output": self.path + "-results-{}.txt".format(i),
                })

            return res
        else:
            display_error("No hosts provided to scan.")
            sys.exit(1)

    def build_cmd(self, args):

        command = self.binary + " -m file-scan -file {target} -out {output} "

        if args.tool_args:
            command += args.tool_args

        return command

    def process_output(self, cmds):
        """
        """

        self.IPAddress.commit()

    def chunks(self, chunkable, n):
        """ Yield successive n-sized chunks from l.
        """
        for i in xrange(0, len(chunkable), n):
            yield chunkable[i:i + n]  # noqa: E203
コード例 #6
0
class Module(ToolTemplate):
    """
    Module for running masscan. Make sure to pass all masscan-specific arguments at the end, after --tool_args

    """

    name = "Masscan"
    binary_name = "masscan"

    def __init__(self, db):
        self.db = db
        self.BaseDomain = BaseDomainRepository(db, self.name)
        self.Domain = DomainRepository(db, self.name)
        self.IPAddress = IPRepository(db, self.name)
        self.Port = PortRepository(db, self.name)

        self.Vulnerability = VulnRepository(db, self.name)
        self.CVE = CVERepository(db, self.name)
        self.ScopeCIDR = ScopeCIDRRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()
        self.options.add_argument(
            "--hosts",
            help=
            "Things to scan separated by a space. DO NOT USE QUOTES OR COMMAS",
            nargs="+",
        )
        self.options.add_argument("--hosts_file", help="File containing hosts")
        self.options.add_argument(
            "-i",
            "--hosts_database",
            help="Use unscanned hosts from the database",
            action="store_true",
        )
        self.options.add_argument("--rescan",
                                  help="Overwrite files without asking",
                                  action="store_true")
        self.options.add_argument(
            "--filename",
            help="Output filename. By default will use the current timestamp.",
        )
        self.options.set_defaults(timeout=None)
        self.options.add_argument(
            "--import_file",
            help="Import results from an Masscan/Nmap XML file.")

    def get_targets(self, args):

        if args.import_file:
            args.no_binary = True
            return [{"target": "", "output": args.import_file}]

        targets = []

        if args.hosts:

            if type(args.hosts) == list:
                for h in args.hosts:
                    if check_if_ip(h):
                        targets.append(h)
                    else:
                        created, domain = self.Domain.find_or_create(domain=h)
                        targets += [i.ip_address for i in domain.ip_addresses]

            else:
                if check_if_ip(h):
                    targets.append(h)
                else:
                    created, domain = self.Domain.find_or_create(domain=h)
                    targets += [i.ip_address for i in domain.ip_addresses]

        if args.hosts_database:
            if args.rescan:
                targets += [
                    h.ip_address
                    for h in self.IPAddress.all(scope_type="active")
                ]
                targets += [h.cidr for h in self.ScopeCIDR.all()]
            else:
                targets += [
                    h.ip_address
                    for h in self.IPAddress.all(tool=self.name,
                                                scope_type="active")
                ]
                targets += [h.cidr for h in self.ScopeCIDR.all(tool=self.name)]

        if args.hosts_file:
            for h in [
                    l for l in open(args.hosts_file).read().split("\n") if l
            ]:
                if check_if_ip(h):
                    targets.append(h)
                else:
                    created, domain = self.Domain.find_or_create(domain=h)
                    targets += [i.ip_address for i in domain.ip_addresses]

        # Here we should deduplicate the targets, and ensure that we don't have IPs listed that also exist inside CIDRs
        data = []
        for t in targets:
            ips = [str(i) for i in list(IPNetwork(t))]
            data += ips

        _, file_name = tempfile.mkstemp()
        open(file_name, "w").write("\n".join(set(data)))

        if args.output_path[0] == "/":
            self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
                                     args.output_path[1:])
        else:
            self.path = os.path.join(self.base_config["PROJECT"]["base_path"],
                                     args.output_path)

        if not os.path.exists(self.path):
            os.makedirs(self.path)

        if args.filename:
            output_path = os.path.join(self.path, args.filename)
        else:
            output_path = os.path.join(
                self.path,
                "masscan-%s.xml" %
                datetime.datetime.now().strftime("%Y.%m.%d-%H.%M.%S"),
            )

        return [{"target": file_name, "output": output_path}]

    def build_cmd(self, args):

        command = "sudo " + self.binary + " -oX {output} -iL {target} "

        if args.tool_args:
            command += args.tool_args

        return command

    def process_output(self, cmds):

        self.import_masscan(cmds[0]["output"])
        if cmds[0]["target"]:
            os.unlink(cmds[0]["target"])

    def import_masscan(
            self,
            filename):  # domains={}, ips={}, rejects=[] == temp while no db
        nFile = filename

        try:
            tree = ET.parse(nFile)
            root = tree.getroot()
            hosts = root.findall("host")

        except Exception:
            print(nFile + " doesn't exist somehow...skipping")
            return

        for host in hosts:
            hostIP = host.find("address").get("addr")

            created, ip = self.IPAddress.find_or_create(ip_address=hostIP)

            for hostname in host.findall("hostnames/hostname"):
                hostname = hostname.get("name")
                hostname = hostname.lower().replace("www.", "")

                # reHostname = re.search(
                #     r"\d{1,3}\-\d{1,3}\-\d{1,3}\-\d{1,3}", hostname
                # )  # attempt to not get PTR record
                # if not reHostname:

                created, domain = self.Domain.find_or_create(domain=hostname)
                if ip not in domain.ip_addresses:
                    domain.ip_addresses.append(ip)
                    domain.save()

            for port in host.findall("ports/port"):

                if port.find("state").get("state"):
                    portState = port.find("state").get("state")
                    hostPort = port.get("portid")
                    portProto = port.get("protocol")

                    created, db_port = self.Port.find_or_create(
                        port_number=hostPort,
                        status=portState,
                        proto=portProto,
                        ip_address=ip,
                    )
                    info = db_port.info
                    if not info:
                        info = {}

                    if port.find("service") is not None:
                        service = port.find("service")
                        portName = service.get("name")
                        if portName == "http" and hostPort == "443":
                            portName = "https"
                        banner = service.get("banner", None)
                        if banner:
                            print("Found banner: {}".format(banner))
                            info["banner"] = banner
                    else:
                        portName = "Unknown"

                    if created:
                        db_port.service_name = portName
                    db_port.info = info
                    db_port.save()

            self.IPAddress.commit()

    def get_domains_from_cert(self, cert):
        # Shamelessly lifted regex from stack overflow
        regex = r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}"

        domains = list(
            set([d for d in re.findall(regex, cert) if "*" not in d]))

        return domains
コード例 #7
0
class Module(ModuleTemplate):
    """
    The Shodan module will either iterate through Shodan search results from net:<cidr>
    for all scoped CIDRs, or a custom search query. The resulting IPs and ports will be
    added to the database, along with a dictionary object of the API results.

    """

    name = "ShodanImport"

    def __init__(self, db):
        self.db = db
        self.Port = PortRepository(db, self.name)
        self.IPAddress = IPRepository(db, self.name)
        self.ScopeCidr = ScopeCIDRRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()

        self.options.add_argument("-k",
                                  "--api_key",
                                  help="API Key for accessing Shodan")
        self.options.add_argument(
            "-s", "--search", help="Custom search string (will use credits)")
        self.options.add_argument(
            "-i",
            "--import_db",
            help="Import scoped IPs from the database",
            action="store_true",
        )
        self.options.add_argument("--rescan",
                                  help="Rescan CIDRs already processed",
                                  action="store_true")
        self.options.add_argument("--fast",
                                  help="Use 'net' filter. (May use credits)",
                                  action="store_true")
        self.options.add_argument(
            "--cidr_only",
            help="Import only CIDRs from database (not individual IPs)",
            action="store_true",
        )

    def run(self, args):

        if not args.api_key:
            display_error("You must supply an API key to use shodan!")
            return

        if args.search:
            ranges = [args.search]

        if args.import_db:
            ranges = []
            if args.rescan:
                if args.fast:
                    ranges += [
                        "net:{}".format(c.cidr) for c in self.ScopeCidr.all()
                    ]
                else:

                    cidrs = [c.cidr for c in self.ScopeCidr.all()]
                    for c in cidrs:
                        ranges += [str(i) for i in IPNetwork(c)]
                if not args.cidr_only:
                    ranges += [
                        "{}".format(i.ip_address)
                        for i in self.IPAddress.all(scope_type="active")
                    ]
            else:
                if args.fast:
                    ranges += [
                        "net:{}".format(c.cidr)
                        for c in self.ScopeCidr.all(tool=self.name)
                    ]
                else:
                    cidrs = [
                        c.cidr for c in self.ScopeCidr.all(tool=self.name)
                    ]
                    for c in cidrs:
                        ranges += [str(i) for i in IPNetwork(c)]
                if not args.cidr_only:
                    ranges += [
                        "{}".format(i.ip_address)
                        for i in self.IPAddress.all(scope_type="active",
                                                    tool=self.name)
                    ]

        api_host_url = "https://api.shodan.io/shodan/host/{}?key={}"
        api_search_url = (
            "https://api.shodan.io/shodan/host/search?key={}&query={}&page={}")
        for r in ranges:

            time.sleep(1)
            if ":" in r:
                display("Doing Shodan search: {}".format(r))
                try:
                    results = json.loads(
                        requests.get(api_search_url.format(args.api_key, r,
                                                           1)).text)
                    if results.get("error") and "request timed out" in results[
                            "error"]:
                        display_warning(
                            "Timeout occurred on Shodan's side.. trying again in 5 seconds."
                        )
                        results = json.loads(
                            requests.get(
                                api_search_url.format(args.api_key, r,
                                                      1)).text)
                except Exception as e:
                    display_error("Something went wrong: {}".format(e))
                    next

                total = len(results["matches"])
                matches = []
                i = 1
                while total > 0:
                    display("Adding {} results from page {}".format(total, i))
                    matches += results["matches"]
                    i += 1
                    try:
                        time.sleep(1)
                        results = json.loads(
                            requests.get(
                                api_search_url.format(args.api_key, r,
                                                      i)).text)
                        if (results.get("error") and "request timed out"
                                in results["error"]  # noqa: W503
                            ):
                            display_warning(
                                "Timeout occurred on Shodan's side.. trying again in 5 seconds."
                            )
                            results = json.loads(
                                requests.get(
                                    api_search_url.format(args.api_key, r,
                                                          1)).text)

                        total = len(results["matches"])

                    except Exception as e:
                        display_error("Something went wrong: {}".format(e))
                        total = 0
                        pdb.set_trace()

                for res in matches:
                    ip_str = res["ip_str"]
                    port_str = res["port"]
                    transport = res["transport"]

                    display("Processing IP: {} Port: {}/{}".format(
                        ip_str, port_str, transport))

                    created, IP = self.IPAddress.find_or_create(
                        ip_address=ip_str)
                    IP.meta["shodan_data"] = results

                    created, port = self.Port.find_or_create(
                        ip_address=IP, port_number=port_str, proto=transport)
                    if created:
                        svc = ""

                        if res.get("ssl", False):
                            svc = "https"
                        elif res.get("http", False):
                            svc = "http"

                        else:
                            svc = ""

                        port.service_name = svc
                    port.status = "open"
                    port.meta["shodan_data"] = res
                    port.save()
            else:

                try:
                    results = json.loads(
                        requests.get(api_host_url.format(r,
                                                         args.api_key)).text)
                except Exception as e:
                    display_error("Something went wrong: {}".format(e))
                    next
                # pdb.set_trace()
                if results.get("data", False):

                    display("{} results found for: {}".format(
                        len(results["data"]), r))

                    for res in results["data"]:
                        ip_str = res["ip_str"]
                        port_str = res["port"]
                        transport = res["transport"]
                        display("Processing IP: {} Port: {}/{}".format(
                            ip_str, port_str, transport))
                        created, IP = self.IPAddress.find_or_create(
                            ip_address=ip_str)
                        IP.meta["shodan_data"] = results

                        created, port = self.Port.find_or_create(
                            ip_address=IP,
                            port_number=port_str,
                            proto=transport)

                        if created:
                            svc = ""

                            if res.get("ssl", False):
                                svc = "https"
                            elif res.get("http", False):
                                svc = "http"

                            else:
                                svc = ""

                            port.service_name = svc
                        port.status = "open"
                        port.meta["shodan_data"] = res
                        port.save()

        self.IPAddress.commit()
コード例 #8
0
class Module(ToolTemplate):
    '''
    This module uses Gowitness to take a screenshot of any discovered web servers. It can be installed from:

    https://github.com/sensepost/gowitness

    '''
    name = "Gowitness"
    binary_name = "gowitness"

    def __init__(self, db):
        self.db = db
        self.IPAddress = IPRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()

        self.options.add_argument(
            "-i",
            "--import_database",
            help="Import URLs from the database",
            action="store_true",
        )
        self.options.add_argument("-f",
                                  "--import_file",
                                  help="Import URLs from file")
        self.options.add_argument(
            "--group_size",
            help="How many hosts per group (default 250)",
            type=int,
            default=250,
        )
        self.options.add_argument(
            "--rescan",
            help="Rerun gowitness on systems that have already been processed.",
            action="store_true",
        )
        self.options.add_argument(
            "--scan_folder",
            help=
            "Generate list of URLs based off of a folder containing GobusterDir output files",
        )
        self.options.add_argument("--counter_max",
                                  help="Max number of screenshots per host",
                                  default="20")

    def get_targets(self, args):

        timestamp = str(int(time()))
        targets = []
        if args.import_file:
            targets += [
                t for t in open(args.import_file).read().split("\n") if t
            ]

        if args.import_database:
            if args.rescan:
                targets += get_urls.run(self.db, scope_type="active")
            else:
                targets += get_urls.run(self.db,
                                        scope_type="active",
                                        tool=self.name)

        if args.scan_folder:

            files = os.listdir(args.scan_folder)
            counter_max = str(args.counter_max)
            for f in files:

                if f.count("_") == 4:
                    counter = 0
                    http, _, _, domain, port = f.split("-dir.txt")[0].split(
                        "_")
                    for data in (open(os.path.join(args.scan_folder,
                                                   f)).read().split("\n")):
                        if "(Status: 200)" in data:
                            targets.append("{}://{}:{}{}".format(
                                http, domain, port,
                                data.split(" ")[0]))
                            counter += 1
                        if counter >= counter_max:
                            break

        if args.output_path[0] == "/":
            self.path = os.path.join(
                self.base_config["PROJECT"]["base_path"],
                args.output_path[1:],
                timestamp,
                args.output_path[1:].split("/")[1] + "_{}",
            )
        else:
            self.path = os.path.join(
                self.base_config["PROJECT"]["base_path"],
                args.output_path,
                timestamp,
                args.output_path.split("/")[1] + "_{}",
            )

        res = []
        i = 0

        for url_chunk in self.chunks(targets, args.group_size):
            i += 1

            _, file_name = tempfile.mkstemp()
            open(file_name, "w").write("\n".join(url_chunk))
            if not os.path.exists(self.path.format(i)):
                os.makedirs(self.path.format(i))
            res.append({"target": file_name, "output": self.path.format(i)})

        return res

    def build_cmd(self, args):

        command = (self.binary +
                   " file -D {output}/gowitness.db -d {output} -s {target} ")

        if args.tool_args:
            command += args.tool_args

        return command

    def process_output(self, cmds):
        """
        Not really any output to process with this module, but you need to cwd into directory to make database generation work, so
        I'll do that here.
        """

        cwd = os.getcwd()
        for cmd in cmds:
            output = cmd["output"]

            cmd = [self.binary, "generate"]
            os.chdir(output)

            Popen(cmd, shell=False).wait()
            os.chdir(cwd)

        self.IPAddress.commit()

    def chunks(self, chunkable, n):
        """ Yield successive n-sized chunks from l.
        """
        for i in xrange(0, len(chunkable), n):
            yield chunkable[i:i + n]  # noqa: E203
コード例 #9
0
ファイル: Ingestor.py プロジェクト: cclauss/armory-1
class Module(ModuleTemplate):
    """
    Ingests domains and IPs. Domains get ip info and cidr info, and IPs get
    CIDR info.

    """

    name = "Ingestor"

    def __init__(self, db):
        self.db = db
        self.BaseDomain = BaseDomainRepository(db, self.name)
        self.Domain = DomainRepository(db, self.name)
        self.IPAddress = IPRepository(db, self.name)
        self.CIDR = CIDRRepository(db, self.name)
        self.ScopeCIDR = ScopeCIDRRepository(db, self.name)

    def set_options(self):
        super(Module, self).set_options()

        self.options.add_argument(
            "-d",
            "--import_domains",
            help="Either domain to import or file containing domains to import. One per line",
        )
        self.options.add_argument(
            "-i",
            "--import_ips",
            help="Either IP/range to import or file containing IPs and ranges, one per line.",
        )
        self.options.add_argument(
            "-a",
            "--active",
            help="Set scoping on imported data as active",
            action="store_true",
        )
        self.options.add_argument(
            "-p",
            "--passive",
            help="Set scoping on imported data as passive",
            action="store_true",
        )
        self.options.add_argument(
            "-sc",
            "--scope_cidrs",
            help="Cycle through out of scope networks and decide if you want to add them in scope",
            action="store_true",
        )
        self.options.add_argument(
            "-sb",
            "--scope_base_domains",
            help="Cycle through out of scope base domains and decide if you want to add them in scope",
            action="store_true",
        )
        self.options.add_argument("--descope", help="Descope an IP, domain, or CIDR")
        self.options.add_argument(
            "-Ii",
            "--import_database_ips",
            help="Import IPs from database",
            action="store_true",
        )
        self.options.add_argument(
            "--force",
            help="Force processing again, even if already processed",
            action="store_true",
        )

    def run(self, args):

        self.in_scope = args.active
        self.passive_scope = args.passive

        if args.descope:
            if "/" in args.descope:
                self.descope_cidr(args.descope)
            elif check_string(args.descope):
                pass

            else:
                self.descope_ip(args.descope)

                # Check if in ScopeCIDR and remove if found

        if args.import_ips:
            try:
                ips = open(args.import_ips)
                for line in ips:

                    if line.strip():
                        if "/" in line or "-" in line:
                            self.process_cidr(line)

                        else:
                            self.process_ip(line.strip(), force_scope=True)
                        self.Domain.commit()
            except IOError:

                if "/" in args.import_ips or "-" in args.import_ips:
                    self.process_cidr(args.import_ips)

                else:
                    self.process_ip(args.import_ips.strip(), force_scope=True)
                self.Domain.commit()

        if args.import_domains:
            try:
                domains = open(args.import_domains)
                for line in domains:
                    if line.strip():
                        self.process_domain(line.strip())
                        self.Domain.commit()
            except IOError:
                self.process_domain(args.import_domains.strip())
                self.Domain.commit()

        if args.scope_base_domains:
            base_domains = self.BaseDomain.all(in_scope=False, passive_scope=False)

            for bd in base_domains:
                self.reclassify_domain(bd)

            self.BaseDomain.commit()

    def get_domain_ips(self, domain):
        ips = []
        try:
            answers = dns.resolver.query(domain, "A")
            for a in answers:
                ips.append(a.address)
            return ips
        except Exception:
            return []

    def process_domain(self, domain_str):

        created, domain = self.Domain.find_or_create(
            only_tool=True,
            domain=domain_str,
            in_scope=self.in_scope,
            passive_scope=self.passive_scope,
        )
        if not created:
            if (
                domain.in_scope != self.in_scope
                or domain.passive_scope != self.passive_scope  # noqa: W503
            ):
                display(
                    "Domain %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
                    % (domain_str, self.in_scope, self.passive_scope)
                )

                domain.in_scope = self.in_scope
                domain.passive_scope = self.passive_scope
                domain.update()

                if domain.base_domain.domain == domain.domain:
                    display("Name also matches a base domain. Updating that as well.")
                    domain.base_domain.in_scope = self.in_scope
                    domain.base_domain.passive_scope = self.passive_scope
                    domain.base_domain.update()

    def process_ip(self, ip_str, force_scope=True):

        created, ip = self.IPAddress.find_or_create(
            only_tool=True,
            ip_address=ip_str,
            in_scope=self.in_scope,
            passive_scope=self.passive_scope,
        )
        if not created:
            if ip.in_scope != self.in_scope or ip.passive_scope != self.passive_scope:
                display(
                    "IP %s already exists with different scoping. Updating to Active Scope: %s Passive Scope: %s"
                    % (ip_str, self.in_scope, self.passive_scope)
                )

                ip.in_scope = self.in_scope
                ip.passive_scope = self.passive_scope
                ip.update()
        return ip

    def process_cidr(self, line):
        display("Processing %s" % line)
        if "/" in line:
            created, cidr = self.ScopeCIDR.find_or_create(cidr=line.strip())
            if created:
                display_new("Adding %s to scoped CIDRs in database" % line.strip())
                cidr.in_scope = True
                cidr.update()

        elif "-" in line:
            start_ip, end_ip = line.strip().replace(" ", "").split("-")
            if "." not in end_ip:
                end_ip = ".".join(start_ip.split(".")[:3] + [end_ip])

            cidrs = iprange_to_cidrs(start_ip, end_ip)

            for c in cidrs:

                created, cidr = self.ScopeCIDR.find_or_create(cidr=str(c))
                if created:
                    display_new("Adding %s to scoped CIDRs in database" % line.strip())
                    cidr.in_scope = True
                    cidr.update()

    def reclassify_domain(self, bd):
        if bd.meta.get("whois", False):
            display_new("Whois data found for {}".format(bd.domain))
            print(bd.meta["whois"])
            res = six.input(
                "Should this domain be scoped (A)ctive, (P)assive, or (N)ot? [a/p/N] "
            )
            if res.lower() == "a":
                bd.in_scope = True
                bd.passive_scope = True

            elif res.lower() == "p":
                bd.in_scope = False
                bd.passive_scope = True
            else:
                bd.in_scope = False
                bd.passive_scope = False
            bd.save()
        else:
            display_error(
                "Unfortunately, there is no whois information for {}. Please populate it using the Whois module".format(
                    bd.domain
                )
            )

    def descope_ip(self, ip):
        ip = self.IPAddress.all(ip_address=ip)
        if ip:
            for i in ip:
                display("Removing IP {} from scope".format(i.ip_address))
                i.in_scope = False
                i.passive_scope = False
                i.update()
                for d in i.domains:
                    in_scope_ips = [
                        ipa
                        for ipa in d.ip_addresses
                        if ipa.in_scope or ipa.passive_scope
                    ]
                    if not in_scope_ips:
                        display(
                            "Domain {} has no more scoped IPs. Removing from scope.".format(
                                d.domain
                            )
                        )
                        d.in_scope = False
                        d.passive_scope = False
            self.IPAddress.commit()

    def descope_cidr(self, cidr):
        CIDR = self.ScopeCIDR.all(cidr=cidr)
        if CIDR:
            for c in CIDR:
                display("Removing {} from ScopeCIDRs".format(c.cidr))
                c.delete()
        cnet = IPNetwork(cidr)
        for ip in self.IPAddress.all():
            if IPAddress(ip.ip_address) in cnet:

                self.descope_ip(ip.ip_address)