Example #1
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included to first determine if there is a 
        robots.txt file present on the website. If that is not found, this
        unit will abort.
        """

        # Run the parent constructor, to ensure this is a valid URL
        super(Unit, self).__init__(*args, **kwargs)

        # Try to get the robots.txt file
        try:
            r = requests.get(
                "{0}/{1}".format(self.target.url_root.rstrip("/"),
                                 "robots.txt"),
                headers=headers,
            )

        except requests.exceptions.ConnectionError:

            raise NotApplicable("cannot reach url")

        # Check if the request succeeded
        if r.status_code != 200:
            # Completely fail if there is nothing there.
            raise NotApplicable("no http 200 response from /robots.txt")

        # Keep track of the response variable for later use
        self.response = r

        # Look for flags in the robots.txt file itself, just in case!
        self.manager.find_flag(self, r.text)
Example #2
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included just to provide a keyword for the
        ``FileUnit``, ensuring the provided target is an image file. This
        also validates it can be read and resizes the image if necessary.
        """
        super(Unit, self).__init__(*args, **kwargs, keywords=[" image "])

        try:
            self.img = Image.open(self.target.path)

            resizing = False
            new_size = list(self.img.size)
            if self.img.size[0] > 1000:
                resizing = True
                new_size[0] = 500
            if self.img.size[1] > 1000:
                resizing = True
                new_size[1] = 500
            if resizing:
                self.img = self.img.resize(tuple(new_size), Image.ANTIALIAS)

            self.data = self.img.load()

        # If we don't know what this is, don't bother with it.
        except OSError:
            raise NotApplicable("cannot read file")

        except Exception:
            raise NotApplicable("unknown error occurred")
Example #3
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        if b"moo" not in self.target.stream:
            raise NotApplicable("no moo in target, must not be cow esolang")

        # We don't run this on URLs
        if self.target.is_url and not self.target.url_accessible:
            raise NotApplicable("URL")
Example #4
0
    def __init__(self, *args, **kwargs):

        super(CryptoUnit, self).__init__(*args, **kwargs)

        # if this is a URL, and we can reach it, don't try to mangle anything
        if self.target.is_url and not self.target.url_accessible:
            raise NotApplicable("this is a URL")

        # if this is a given file, ensure it's not an image or anything useful
        if self.target.path:
            if is_good_magic(magic.from_file(self.target.path)):
                raise NotApplicable("potentially useful file")
Example #5
0
    def __init__(self, manager: Manager, target: Target):
        super(Unit, self).__init__(manager, target)

        # Ensure the target is printable data
        if not self.target.is_printable:
            raise NotApplicable("not printable data")

        # Ensure the target is not english
        if self.target.is_english:
            raise NotApplicable("seemingly english")

        # if this was a given file, make sure it's not an image or anything useful
        if self.target.path:
            if is_good_magic(magic.from_file(self.target.path)):
                raise NotApplicable("potentially useful file")
Example #6
0
    def __init__(self, *args, **kwargs):
        """
        The constructor collects the supplied key and creates the 
        appropriate alphabets.
        """

        super(Unit, self).__init__(*args, **kwargs)

        # First, ensure a key has been supplied to the unit.
        self.key = self.get("key", default=None)
        if self.key is None:
            raise NotApplicable("no key is supplied to run keyed caesar")

        # Now begin to prepare the alphabets.
        self.lower_alphabet = string.ascii_lowercase
        self.upper_alphabet = string.ascii_uppercase

        # Remove the seen letters in the alphabets
        for letter in self.key[::-1]:
            # Do this for the lower-case rendition of the key and alphabet
            if letter.lower() in self.lower_alphabet:
                self.lower_alphabet = self.lower_alphabet.replace(
                    letter.lower(), "")
                self.lower_alphabet = letter.lower() + self.lower_alphabet

            # Also for the upper-case rendition of the key and alphabet
            if letter.upper() in self.upper_alphabet:
                self.upper_alphabet = self.upper_alphabet.replace(
                    letter.upper(), "")
                self.upper_alphabet = letter.upper() + self.upper_alphabet
Example #7
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included to first determine if there is a form
        on this web page. If no form is found, it will abort.
        """

        # Run the parent constructor, to ensure this is a valid URL
        super(Unit, self).__init__(*args, **kwargs)

        self.action = re.findall(
            rb'<\s*form.*action\s*=\s*[\'"](.+?)[\'"]',
            self.target.raw,
            flags=re.IGNORECASE,
        )
        self.method = re.findall(
            rb'<\s*form.*method\s*=\s*[\'"](.+?)[\'"]',
            self.target.raw,
            flags=re.IGNORECASE,
        )

        self.username = re.findall(web.user_regex,
                                   self.target.raw,
                                   flags=re.IGNORECASE)
        self.password = re.findall(web.pass_regex,
                                   self.target.raw,
                                   flags=re.IGNORECASE)

        # Sometimes, a form might not have an explicit location. Assume the current page!
        if self.action == []:
            self.action = [b"#"]

        # Only run this if we have potential information...
        if not (self.action and self.method and self.username
                and self.password):
            raise NotApplicable("no form found")
Example #8
0
    def __init__(self, *args, **kwargs):

        super(WebUnit, self).__init__(*args, **kwargs)

        if not self.target.is_url:

            raise NotApplicable("not a web url")
Example #9
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        # if this was a file, ensure it's not an image or anything useful
        if self.target.path:
            if is_good_magic(magic.from_file(self.target.path)):
                raise NotApplicable("potentially useful file")
Example #10
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        # Find matches in the target
        self.matches: list = MD5_PATTERN.findall(self.target.raw)

        if self.matches is None or len(self.matches) == 0:
            raise NotApplicable("No md5 hashes found")
Example #11
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        matches = OOK_REGEX.findall(self.target.raw)

        if matches is None or matches == []:
            raise NotApplicable("no ook potential found")
        else:
            self.code = b"".join([m[-1] for m in matches])
Example #12
0
    def __init__(self, *args, **kwargs):
        super(PrintableDataUnit, self).__init__(*args, **kwargs)

        # Grab the pikalang commands from the targets
        self.pika_commands = re.findall(bytes(regex_finder, "utf-8"),
                                        self.target.raw)

        if len(self.pika_commands) <= 5:
            raise NotApplicable("not enough pikalang")
Example #13
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included just to provide a keyword for the
        ``FileUnit``, ensuring the provided target is in fact a PDF file.
        """
        super(Unit, self).__init__(*args, **kwargs, keywords=["pdf document"])

        # Check to see if this PDF is even password protected
        try:
            with open(self.target.path, "rb") as f:
                pdf = PdfFileReader(f)
                if not pdf.isEncrypted:
                    raise NotApplicable("pdf is not encrypted")
        except NotApplicable as e:
            # This is here to raise this NotApplicable up, in case it fails
            # before by opening the PDF file
            raise e
        except:
            raise NotApplicable("failed to open/read file")
Example #14
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        self.c = -1
        self.e = parse_int(self.get("e"))
        self.n = parse_int(self.get("n"))
        self.q = parse_int(self.get("q"))
        self.p = parse_int(self.get("p"))
        self.d = parse_int(self.get("d"))
        self.dq = parse_int(self.get("dq"))
        self.dp = parse_int(self.get("dp"))
        self.phi = parse_int(self.get("phi"))

        if self.get("c"):
            try:
                handle = open(self.get("c"), "rb")
                is_file = True
            except OSError:
                is_file = False

            if is_file:
                ciphertext_data = handle.read()
                self.c = parse_int(ciphertext_data)
                if self.c == -1:
                    raise NotApplicable(
                        "could not determine ciphertext from file")
            else:
                self.c = parse_int(self.get("c"))

        if self.target.is_file:
            try:
                self.raw_target = self.target.stream.read().decode("utf-8")
            except UnicodeDecodeError:
                raise NotApplicable(
                    "unicode error, must not be potential ciphertext")

            for finding in find_variables(self.raw_target):
                if finding:
                    vars(self)[finding[0]] = parse_int(finding[1])

        if self.c == -1:
            raise NotApplicable("no ciphertext determined")
Example #15
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        # Ensure this is printable data
        if not self.target.is_printable:
            raise NotApplicable("not printable data")

        # Ensure this is not english data
        if self.target.is_english:
            raise NotApplicable("seemingly english")

        # if this was a file, ensure it's not an image or anything useful
        if self.target.path:
            if is_good_magic(magic.from_file(self.target.path)):
                raise NotApplicable("potentially useful file")

        # Are there base32 chunks in the data?
        self.matches = BASE32_REGEX.findall(self.target.raw)
        if self.matches is None:
            raise NotApplicable("no base32 text found")
Example #16
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included just to provide a keyword for the
        ``FileUnit``, ensuring the provided target is an audio file. This
        also validates it is not a URL to a website.
        """

        super(Unit, self).__init__(*args, **kwargs, keywords=["audio"])

        # if this is a URL, and we can reach it, don't try to mangle anything
        if self.target.is_url and not self.target.url_accessible:
            raise NotApplicable("this is a URL")
Example #17
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)
        """
        The constructor is included just to provide a keyword for the
        ``FileUnit``, ensuring the provided target is a file. This
        also validates if there tabs or spaces found inside the target.
        """

        self.space_pieces = re.findall(b"[ \t]+", self.target.raw)

        if self.space_pieces is None or self.space_pieces == []:
            raise NotApplicable("no spaces found")
Example #18
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)
        try:
            self.raw_target = self.target.stream.read().decode("utf-8")
        except UnicodeDecodeError:
            raise units.NotApplicable(
                "unicode error, unlikely usable cryptogram")

        try:
            requests.get("https://quipqiup.com/", verify=False)
        except requests.exceptions.ConnectionError:
            raise NotApplicable("cannot reach quipqiup solver")
Example #19
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included to first determine if a found target
        is an attachment or a bad MIME type. If this is the case, the unit
        will abort.
        """
        super(Unit, self).__init__(*args, **kwargs)

        # avoid attachments
        if "Content-Disposition" in self.target.request.headers:
            if "attachment" in self.target.request.headers[
                    "Content-Disposition"]:
                raise NotApplicable("spider cannot handle attachments")

        # avoid bad mime types
        if "Content-Type" in self.target.request.headers:
            content_type = self.target.request.headers["Content-Type"].lower()
            for bad_type in self.BAD_MIME_TYPES:
                if bad_type in content_type:
                    raise NotApplicable(
                        "spider does not support {0} files".format(bad_type))
Example #20
0
    def __init__(self, *args, **kwargs):
        """
        The constructor is included just to provide a keyword for the
        ``FileUnit``, ensuring the provided target is an audio file. This
        also validates it is a .wav file (.mp3 is not yet supported)
        """

        super(Unit, self).__init__(*args, **kwargs, keywords=["audio"])

        # Create a detector object
        self.detector = DTMFdetector()

        try:
            # Ensure this will even work
            if isinstance(self.target.path, bytes):
                self.detector.check(self.target.path.decode("utf-8"))
            else:
                self.detector.check(self.target.path)
        except wave.Error:
            raise NotApplicable(
                "no RIFF id... not a .wav? mp3 not yet supported..")
        except:
            raise NotApplicable("failure reading dtmf tones")
Example #21
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)
        try:
            self.raw_target = self.target.stream.read().decode("utf-8")
        except UnicodeDecodeError:
            raise units.NotApplicable(
                "unicode error, unlikely usable cryptogram")

        try:
            requests.get(
                "https://6n9n93nlr5.execute-api.us-east-1.amazonaws.com/prod/solve"
            )
        except requests.exceptions.ConnectionError:
            raise NotApplicable("cannot reach quipqiup solver")
Example #22
0
    def __init__(self, manager: Manager, target: Target):
        """
        The constructor is included just to provide a keyword for the
        ``FileUnit``, ensuring the provided target is in fact an image. This
        also validates it can open the file with PIL without an issue.
        """

        super(Unit, self).__init__(manager, target, keywords="image")

        try:
            # Attempt to open the image with PIL
            self.image = Image.open(self.target.path)
        except OSError:
            raise NotApplicable("not an image")
Example #23
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        # Grab the configuration data
        self.git_proxy = self.get("proxy", default="")
        self.git_jobs = self.geti("jobs", default=10)
        self.git_timeout = self.geti("git_timeout", default=3)
        self.git_retry = self.geti("git_retry", default=3)
        self.flag_format = self.manager.flag_pattern.pattern.decode("utf-8")

        # Validate these configs to ensure they make sense
        if self.git_jobs < 1:
            raise NotApplicable("invalid number of git-jobs")

        # retry validation
        if self.git_retry < 1:
            raise NotApplicable("invalid number of git-retries")

        # timeout validation
        if self.git_timeout < 1:
            raise NotApplicable("invalid git timeout")

        # proxy validation
        if self.git_proxy:
            proxy_valid = False

            for pattern, proxy_type in [
                (r"^socks5:(.*):(\d+)$", socks.PROXY_TYPE_SOCKS5),
                (r"^socks4:(.*):(\d+)$", socks.PROXY_TYPE_SOCKS4),
                (r"^http://(.*):(\d+)$", socks.PROXY_TYPE_HTTP),
                (r"^(.*):(\d+)$", socks.PROXY_TYPE_SOCKS5),
            ]:
                m = re.match(pattern, self.git_proxy)
                if m:
                    socks.setdefaultproxy(proxy_type, m.group(1),
                                          int(m.group(2)))
                    socket.socket = socks.socksocket
                    proxy_valid = True
                    break

            if not proxy_valid:
                raise NotApplicable("invalid git proxy")

        # Try to get see if there is a .git directory
        url = "{0}/{1}".format(self.target.url_root.rstrip("/"), ".git/HEAD")

        try:
            r = requests.get(url, allow_redirects=False)
        except (requests.exceptions.ConnectionError, ):
            raise NotApplicable("cannot reach server")

        # If the response is anything other than a "Not Found",
        # we might have something here...
        if r.status_code == 404:
            raise NotApplicable("http response 404 at /.git/HEAD")
        else:
            self.response = r
Example #24
0
    def __init__(self, *args, **kwargs):
        """
        The constructor verifies that that only DNA letters (A, T, C, G) 
        are found in the target.
        """

        super(Unit, self).__init__(*args, **kwargs)

        self.raw_target = self.target.stream.read().decode("utf-8").upper()
        self.raw_target = self.raw_target.replace(" ",
                                                  "").replace("U",
                                                              "T").strip()

        if (not all(c in "ACGT" for c in self.raw_target)
                or len(self.raw_target) % 3 != 0):

            raise NotApplicable("more than DNA letters (A, T, C, G) found")
Example #25
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        self.jsfuck = re.findall(rb"[\\[\\(\\)\\+!\]]{5,}", self.target.raw)
        if not self.jsfuck:
            raise NotApplicable("no jsfuck code found")
Example #26
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        if URL_DATA.search(self.target.raw) is None:
            raise NotApplicable("No URL encoded parts")
Example #27
0
    def __init__(self, *args, **kwargs):
        super(Unit, self).__init__(*args, **kwargs)

        self.vignere_key = self.get("key")
        if not self.vignere_key:
            raise NotApplicable("empty vigenere key passed")