def __init__(self, *args, **kwargs):
"""
The constructor is included to first determine if there is a
robots.txt file present on the website. If that is not found, this
unit will abort.
"""
# Run the parent constructor, to ensure this is a valid URL
super(Unit, self).__init__(*args, **kwargs)
# Try to get the robots.txt file
try:
r = requests.get(
"{0}/{1}".format(self.target.url_root.rstrip("/"),
"robots.txt"),
headers=headers,
)
except requests.exceptions.ConnectionError:
raise NotApplicable("cannot reach url")
# Check if the request succeeded
if r.status_code != 200:
# Completely fail if there is nothing there.
raise NotApplicable("no http 200 response from /robots.txt")
# Keep track of the response variable for later use
self.response = r
# Look for flags in the robots.txt file itself, just in case!
self.manager.find_flag(self, r.text)
def __init__(self, *args, **kwargs):
"""
The constructor is included just to provide a keyword for the
``FileUnit``, ensuring the provided target is an image file. This
also validates it can be read and resizes the image if necessary.
"""
super(Unit, self).__init__(*args, **kwargs, keywords=[" image "])
try:
self.img = Image.open(self.target.path)
resizing = False
new_size = list(self.img.size)
if self.img.size[0] > 1000:
resizing = True
new_size[0] = 500
if self.img.size[1] > 1000:
resizing = True
new_size[1] = 500
if resizing:
self.img = self.img.resize(tuple(new_size), Image.ANTIALIAS)
self.data = self.img.load()
# If we don't know what this is, don't bother with it.
except OSError:
raise NotApplicable("cannot read file")
except Exception:
raise NotApplicable("unknown error occurred")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
if b"moo" not in self.target.stream:
raise NotApplicable("no moo in target, must not be cow esolang")
# We don't run this on URLs
if self.target.is_url and not self.target.url_accessible:
raise NotApplicable("URL")
def __init__(self, *args, **kwargs):
super(CryptoUnit, self).__init__(*args, **kwargs)
# if this is a URL, and we can reach it, don't try to mangle anything
if self.target.is_url and not self.target.url_accessible:
raise NotApplicable("this is a URL")
# if this is a given file, ensure it's not an image or anything useful
if self.target.path:
if is_good_magic(magic.from_file(self.target.path)):
raise NotApplicable("potentially useful file")
def __init__(self, manager: Manager, target: Target):
super(Unit, self).__init__(manager, target)
# Ensure the target is printable data
if not self.target.is_printable:
raise NotApplicable("not printable data")
# Ensure the target is not english
if self.target.is_english:
raise NotApplicable("seemingly english")
# if this was a given file, make sure it's not an image or anything useful
if self.target.path:
if is_good_magic(magic.from_file(self.target.path)):
raise NotApplicable("potentially useful file")
def __init__(self, *args, **kwargs):
"""
The constructor collects the supplied key and creates the
appropriate alphabets.
"""
super(Unit, self).__init__(*args, **kwargs)
# First, ensure a key has been supplied to the unit.
self.key = self.get("key", default=None)
if self.key is None:
raise NotApplicable("no key is supplied to run keyed caesar")
# Now begin to prepare the alphabets.
self.lower_alphabet = string.ascii_lowercase
self.upper_alphabet = string.ascii_uppercase
# Remove the seen letters in the alphabets
for letter in self.key[::-1]:
# Do this for the lower-case rendition of the key and alphabet
if letter.lower() in self.lower_alphabet:
self.lower_alphabet = self.lower_alphabet.replace(
letter.lower(), "")
self.lower_alphabet = letter.lower() + self.lower_alphabet
# Also for the upper-case rendition of the key and alphabet
if letter.upper() in self.upper_alphabet:
self.upper_alphabet = self.upper_alphabet.replace(
letter.upper(), "")
self.upper_alphabet = letter.upper() + self.upper_alphabet
def __init__(self, *args, **kwargs):
"""
The constructor is included to first determine if there is a form
on this web page. If no form is found, it will abort.
"""
# Run the parent constructor, to ensure this is a valid URL
super(Unit, self).__init__(*args, **kwargs)
self.action = re.findall(
rb'<\s*form.*action\s*=\s*[\'"](.+?)[\'"]',
self.target.raw,
flags=re.IGNORECASE,
)
self.method = re.findall(
rb'<\s*form.*method\s*=\s*[\'"](.+?)[\'"]',
self.target.raw,
flags=re.IGNORECASE,
)
self.username = re.findall(web.user_regex,
self.target.raw,
flags=re.IGNORECASE)
self.password = re.findall(web.pass_regex,
self.target.raw,
flags=re.IGNORECASE)
# Sometimes, a form might not have an explicit location. Assume the current page!
if self.action == []:
self.action = [b"#"]
# Only run this if we have potential information...
if not (self.action and self.method and self.username
and self.password):
raise NotApplicable("no form found")
def __init__(self, *args, **kwargs):
super(WebUnit, self).__init__(*args, **kwargs)
if not self.target.is_url:
raise NotApplicable("not a web url")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
# if this was a file, ensure it's not an image or anything useful
if self.target.path:
if is_good_magic(magic.from_file(self.target.path)):
raise NotApplicable("potentially useful file")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
# Find matches in the target
self.matches: list = MD5_PATTERN.findall(self.target.raw)
if self.matches is None or len(self.matches) == 0:
raise NotApplicable("No md5 hashes found")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
matches = OOK_REGEX.findall(self.target.raw)
if matches is None or matches == []:
raise NotApplicable("no ook potential found")
else:
self.code = b"".join([m[-1] for m in matches])
def __init__(self, *args, **kwargs):
super(PrintableDataUnit, self).__init__(*args, **kwargs)
# Grab the pikalang commands from the targets
self.pika_commands = re.findall(bytes(regex_finder, "utf-8"),
self.target.raw)
if len(self.pika_commands) <= 5:
raise NotApplicable("not enough pikalang")
def __init__(self, *args, **kwargs):
"""
The constructor is included just to provide a keyword for the
``FileUnit``, ensuring the provided target is in fact a PDF file.
"""
super(Unit, self).__init__(*args, **kwargs, keywords=["pdf document"])
# Check to see if this PDF is even password protected
try:
with open(self.target.path, "rb") as f:
pdf = PdfFileReader(f)
if not pdf.isEncrypted:
raise NotApplicable("pdf is not encrypted")
except NotApplicable as e:
# This is here to raise this NotApplicable up, in case it fails
# before by opening the PDF file
raise e
except:
raise NotApplicable("failed to open/read file")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
self.c = -1
self.e = parse_int(self.get("e"))
self.n = parse_int(self.get("n"))
self.q = parse_int(self.get("q"))
self.p = parse_int(self.get("p"))
self.d = parse_int(self.get("d"))
self.dq = parse_int(self.get("dq"))
self.dp = parse_int(self.get("dp"))
self.phi = parse_int(self.get("phi"))
if self.get("c"):
try:
handle = open(self.get("c"), "rb")
is_file = True
except OSError:
is_file = False
if is_file:
ciphertext_data = handle.read()
self.c = parse_int(ciphertext_data)
if self.c == -1:
raise NotApplicable(
"could not determine ciphertext from file")
else:
self.c = parse_int(self.get("c"))
if self.target.is_file:
try:
self.raw_target = self.target.stream.read().decode("utf-8")
except UnicodeDecodeError:
raise NotApplicable(
"unicode error, must not be potential ciphertext")
for finding in find_variables(self.raw_target):
if finding:
vars(self)[finding[0]] = parse_int(finding[1])
if self.c == -1:
raise NotApplicable("no ciphertext determined")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
# Ensure this is printable data
if not self.target.is_printable:
raise NotApplicable("not printable data")
# Ensure this is not english data
if self.target.is_english:
raise NotApplicable("seemingly english")
# if this was a file, ensure it's not an image or anything useful
if self.target.path:
if is_good_magic(magic.from_file(self.target.path)):
raise NotApplicable("potentially useful file")
# Are there base32 chunks in the data?
self.matches = BASE32_REGEX.findall(self.target.raw)
if self.matches is None:
raise NotApplicable("no base32 text found")
def __init__(self, *args, **kwargs):
"""
The constructor is included just to provide a keyword for the
``FileUnit``, ensuring the provided target is an audio file. This
also validates it is not a URL to a website.
"""
super(Unit, self).__init__(*args, **kwargs, keywords=["audio"])
# if this is a URL, and we can reach it, don't try to mangle anything
if self.target.is_url and not self.target.url_accessible:
raise NotApplicable("this is a URL")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
"""
The constructor is included just to provide a keyword for the
``FileUnit``, ensuring the provided target is a file. This
also validates if there tabs or spaces found inside the target.
"""
self.space_pieces = re.findall(b"[ \t]+", self.target.raw)
if self.space_pieces is None or self.space_pieces == []:
raise NotApplicable("no spaces found")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
try:
self.raw_target = self.target.stream.read().decode("utf-8")
except UnicodeDecodeError:
raise units.NotApplicable(
"unicode error, unlikely usable cryptogram")
try:
requests.get("https://quipqiup.com/", verify=False)
except requests.exceptions.ConnectionError:
raise NotApplicable("cannot reach quipqiup solver")
def __init__(self, *args, **kwargs):
"""
The constructor is included to first determine if a found target
is an attachment or a bad MIME type. If this is the case, the unit
will abort.
"""
super(Unit, self).__init__(*args, **kwargs)
# avoid attachments
if "Content-Disposition" in self.target.request.headers:
if "attachment" in self.target.request.headers[
"Content-Disposition"]:
raise NotApplicable("spider cannot handle attachments")
# avoid bad mime types
if "Content-Type" in self.target.request.headers:
content_type = self.target.request.headers["Content-Type"].lower()
for bad_type in self.BAD_MIME_TYPES:
if bad_type in content_type:
raise NotApplicable(
"spider does not support {0} files".format(bad_type))
def __init__(self, *args, **kwargs):
"""
The constructor is included just to provide a keyword for the
``FileUnit``, ensuring the provided target is an audio file. This
also validates it is a .wav file (.mp3 is not yet supported)
"""
super(Unit, self).__init__(*args, **kwargs, keywords=["audio"])
# Create a detector object
self.detector = DTMFdetector()
try:
# Ensure this will even work
if isinstance(self.target.path, bytes):
self.detector.check(self.target.path.decode("utf-8"))
else:
self.detector.check(self.target.path)
except wave.Error:
raise NotApplicable(
"no RIFF id... not a .wav? mp3 not yet supported..")
except:
raise NotApplicable("failure reading dtmf tones")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
try:
self.raw_target = self.target.stream.read().decode("utf-8")
except UnicodeDecodeError:
raise units.NotApplicable(
"unicode error, unlikely usable cryptogram")
try:
requests.get(
"https://6n9n93nlr5.execute-api.us-east-1.amazonaws.com/prod/solve"
)
except requests.exceptions.ConnectionError:
raise NotApplicable("cannot reach quipqiup solver")
def __init__(self, manager: Manager, target: Target):
"""
The constructor is included just to provide a keyword for the
``FileUnit``, ensuring the provided target is in fact an image. This
also validates it can open the file with PIL without an issue.
"""
super(Unit, self).__init__(manager, target, keywords="image")
try:
# Attempt to open the image with PIL
self.image = Image.open(self.target.path)
except OSError:
raise NotApplicable("not an image")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
# Grab the configuration data
self.git_proxy = self.get("proxy", default="")
self.git_jobs = self.geti("jobs", default=10)
self.git_timeout = self.geti("git_timeout", default=3)
self.git_retry = self.geti("git_retry", default=3)
self.flag_format = self.manager.flag_pattern.pattern.decode("utf-8")
# Validate these configs to ensure they make sense
if self.git_jobs < 1:
raise NotApplicable("invalid number of git-jobs")
# retry validation
if self.git_retry < 1:
raise NotApplicable("invalid number of git-retries")
# timeout validation
if self.git_timeout < 1:
raise NotApplicable("invalid git timeout")
# proxy validation
if self.git_proxy:
proxy_valid = False
for pattern, proxy_type in [
(r"^socks5:(.*):(\d+)$", socks.PROXY_TYPE_SOCKS5),
(r"^socks4:(.*):(\d+)$", socks.PROXY_TYPE_SOCKS4),
(r"^http://(.*):(\d+)$", socks.PROXY_TYPE_HTTP),
(r"^(.*):(\d+)$", socks.PROXY_TYPE_SOCKS5),
]:
m = re.match(pattern, self.git_proxy)
if m:
socks.setdefaultproxy(proxy_type, m.group(1),
int(m.group(2)))
socket.socket = socks.socksocket
proxy_valid = True
break
if not proxy_valid:
raise NotApplicable("invalid git proxy")
# Try to get see if there is a .git directory
url = "{0}/{1}".format(self.target.url_root.rstrip("/"), ".git/HEAD")
try:
r = requests.get(url, allow_redirects=False)
except (requests.exceptions.ConnectionError, ):
raise NotApplicable("cannot reach server")
# If the response is anything other than a "Not Found",
# we might have something here...
if r.status_code == 404:
raise NotApplicable("http response 404 at /.git/HEAD")
else:
self.response = r
def __init__(self, *args, **kwargs):
"""
The constructor verifies that that only DNA letters (A, T, C, G)
are found in the target.
"""
super(Unit, self).__init__(*args, **kwargs)
self.raw_target = self.target.stream.read().decode("utf-8").upper()
self.raw_target = self.raw_target.replace(" ",
"").replace("U",
"T").strip()
if (not all(c in "ACGT" for c in self.raw_target)
or len(self.raw_target) % 3 != 0):
raise NotApplicable("more than DNA letters (A, T, C, G) found")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
self.jsfuck = re.findall(rb"[\\[\\(\\)\\+!\]]{5,}", self.target.raw)
if not self.jsfuck:
raise NotApplicable("no jsfuck code found")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
if URL_DATA.search(self.target.raw) is None:
raise NotApplicable("No URL encoded parts")
def __init__(self, *args, **kwargs):
super(Unit, self).__init__(*args, **kwargs)
self.vignere_key = self.get("key")
if not self.vignere_key:
raise NotApplicable("empty vigenere key passed")