pentestlib2.py

"""

Ensemble de fonctions utiles pour les pentests.
Marche mieux pour Python 2.6+, marche quasi pas avec Python 3.x

"""

##################################################
#
# imports
#


try :
    import re
    from zlib import decompress as GUNZIP, error as ZLIB_ERROR
    from urlparse import urlparse
    from socket import socket, error as SOCKET_ERROR, SOCK_STREAM, AF_INET
    from ssl import wrap_socket
    from re import compile, match
    from sys import stdout, stderr, version_info
    from datetime import datetime
    from httplib import HTTPSConnection, HTTPConnection
    from inspect import stack
    from random import randint
    from itertools import izip, cycle
except ImportError, ie:
    print("[-] Failed on import: %s" % ie)
    exit(1)

except ImportWarning, iw:
    print("[-] Critical: %s" % iw)
    exit(1)

major, minor = version_info[0:2]
if major < 2 : raise ImportError ("Python version must be at least 2.7+")
if minor < 7 : raise ImportWarning ("Python version must be at least 2.7+")


##################################################
#
#  HTTP/SQL/HTML Part
#

class HTTPWrapper :
    """
    Wrapper pour les connexions HTTP. Gere toute la session HTTP. Peut etre
    utilise dans l'interpreteur ou importe dans un script.

    Utilisation basique
    >>> h = HTTPWrapper("https://www.google.fr", proxy="localhost:8080")
    >>> h.execute()
    >>> print h.response_headers
    >>> print h.response_body

    Features:
    - gestion du deflate par fingerprint (plus par header)
    - la gestion du format data json
    - gere le proxy http normal et ssl
    - raw fuzzing functions
    - http keep-alive

    Todo:
    - ameliorer le fuzz (base sur dico, etc.)
    """


    def __init__(self, url, method="GET", proxy=None):
        """ Initialize HTTPWrapper object, parse l'url, declare les fonctions obsoletes """
        self.__name__ = "HTTPWrapper"
        o = urlparse(url, "http")
        self.method = method
        self.proto = o.scheme

        if self.proto == "https":
            self.ssl = True
            self.http = HTTPSConnection(o.netloc, timeout = 15)
        else :
            self.ssl = False
            self.http = HTTPConnection(o.netloc, timeout = 15)

        self.host, self.port = spliturl(o.netloc, self.ssl)
        if o.path == '':
            self.path = '/'
        else :
            self.path = o.path
        self.proxy = proxy

        self._headers = {}
        self._get = []
        self._post = []
        self._response = {}
        self._format = "plain"
        self._keep_alive = False

        self.setbasicheaders()
        for elt in o.query.split("&"):
            if len(elt) == 0: continue
            try:
                k,v = elt.split("=", 1)
            except ValueError:
                k = elt
                v = "1"
            self.addparam(k, html_decode(v))


    def execute(self):
        """
        execute la requete et stocke les resultats dans self.response_headers et
        self.response_body
        """

        if len(self._post) > 0:
            self.method = "POST"

        if self.method == "POST":
            self.addheader("Content-Length", "%d" % len(self.post()))
            self.addheader("Content-Type", "application/x-www-form-urlencoded")

        if self.http.sock is None and self.proxy:
            self.tunnel_connect()

        self.http.request(method  = self.method,
                          url     = self.geturl(),
                          body    = self.post(),
                          headers = self._headers)

        try:

            res = self.http.getresponse()
            self._response = {
                "headers": {
                    "version": res.version,
                    "status": res.status,
                    "reason": res.reason,
                    },
                "body": res.read(),
                }

            self._response["headers"].update( [(k, v) for k,v in res.getheaders()] )
            self.response_headers = self._response_headers()
            self.response_body = self._response_body()

        except Exception, e:
            ERR("Exception in %s: %s" % (called(), e))

        finally:
            if not self._keep_alive or \
                    ("connection" in self.response_headers.keys() \
                         and self.response_headers["connection"] == "close") :
                self.close()


    def close(self):
        if self.http.sock is not None :
            self.http.close()
            self.http.sock = None


    def tunnel_connect(self):
        proxy_host, proxy_port = spliturl(self.proxy, self.ssl)
        proxy_port = int(proxy_port)


        try :

            s = socket(AF_INET, SOCK_STREAM)
            s.connect((proxy_host, proxy_port))

            if self.ssl:

                buf = "CONNECT %s:%s HTTP/1.0\r\n" % (self.host, self.port)
                buf+= "Host: %s\r\n" % self.host
                buf+= "User-Agent: Python HTTPWrapper\r\n"
                buf+= "\r\n"

                s.sendall(buf)
                line = s.recv(256)

                proto, code, msg = line.split(" ", 2)
                code = int(code)

                if code != 200 and not msg.lower().startswith("connection established") :
                    raise SOCKET_ERROR ("Proxy denied (%d) our request: %s" % (code, msg))

                s = wrap_socket(s, self.http.key_file, self.http.cert_file)


        except SOCKET_ERROR, msg:
            ERR ("%s: Failed to proxy, using direct link. Reason: %s" % (called(), msg))
            s.close()
            del (s)
            self.proxy = None
            s = socket(AF_INET, SOCK_STREAM)
            s.connect((self.host, self.port))

            if self.ssl:
                s = wrap_socket(s, self.http.key_file, self.http.cert_file)

        self.close()
        self.http.sock = s
        return


    def geturl(self):
        """ @return l'url complete """
        if self.proxy is None:
            return "%s%s" % (self.path, self.get())
        else:
            return "%s://%s:%d%s%s" % (self.proto,
                                       self.host,
                                       self.port,
                                       self.path,
                                       self.get())

    def addheader(self, param, value):
        """ ajoute un header. si le header est un Cookie, il sera concatene. """
        if param in self._headers and param == "Cookie" :
            self._headers[param] += ";" + value
        else :
            self._headers[param] = value


    def setbasicheaders(self):
        """ add few basic headers """
        self.addheader("Host", self.host)
        self.addheader("User-Agent", "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 6.0; fr-FR)")
        self.addheader("Accept-Encoding", "text/plain")
        self.addheader("Connection", "Close")
        # self.addheader("Accept-Encoding", "gzip,deflate")


    def addparam(self, key, value):
        found = False
        if key in (None, ""):
            return
        ptr = None
        if self.method == "GET" :
            ptr = self._get
        elif self.method == "POST":
            ptr = self._post
        else:
            return

        for a in ptr :
            k,v = a
            if k == key :
                a[1] = value
                found = True

        if not found :
            ptr.append([key, value.strip()])


    def getparams(self, tab):
        if self._format == "json":
            parts = [ "%s:%s" % (k,v) for k,v in tab]
            return '{' + ','.join(parts) + '}'
        else:
            return "&".join([ "%s=%s" % (html_encode(k), html_encode(v)) for k,v in tab])


    def get(self):
        if len(self._get) == 0: return ""
        return "?" + self.getparams(self._get)


    def post(self):
        if (len(self._post) == 0): return ""
        return self.getparams(self._post)


    def _response_headers(self):
        """ @return le hash des headers retournes par le serveur HTTP  """
        try :
            return self._response['headers']
        except :
            return "No headers found"


    def _response_body(self):
        """
        @return string with HTTP response body.
        Now uses magic number instead of being based on HTTP Header "Content-Encoding" (NC)
        """
        try:
            body = self._response['body']
        except KeyError:
            body = "No body found"

        if body.startswith("\x1f\x8b\x08") :
            try:
                body = GUNZIP(body)

            except ZLIB_ERROR, e :
                ERR("Failed to gunzip: %s" % e)
                body = ""

        return body


    def __str__(self):
        """ @return string with object properties """
        chaine = "Target: %s\n" % self.host
        if self.proxy:
            chaine += "Using proxy: %s\n" % self.proxy
        if self.ssl:
            chaine += "Using SSL\n"

        chaine += "%s %s HTTP/1.1\n" % (self.method, self.geturl())
        for k,v in self._headers.iteritems():
            chaine += "%s: %s\n" % (k,v)

        chaine += "\n%s\n" % self.post()

        return chaine


    def __desc__(self):
        """ @return object description """
        return self.__str__()


    def set_keep_alive(self, boolean):
        if boolean not in (True, False):
            return

        if boolean:
            self.addheader("Connection", "Keep-Alive")
        else:
            self.addheader("Connection", "Close")


    def fuzz(self, obj) :
        """ HTTPWrapper object random fuzz """
        try:
            getattr(self, obj)
        except AttributeError :
            ERR("%s has no attribute '%s'" % (self.__name__, obj))
            return

        if obj in ("host", "port"):
            ERR("Cannot fuzz '%s' object" % obj)
            return

        def _fuzz(o):
            FUZZ_SPACE = 256
            FUZZ_MAXLEN = 50
            lg = randint(0, FUZZ_MAXLEN)
            c = ""
            while lg:
                c += chr(randint(0, FUZZ_SPACE))
                lg-= 1

            return c

        type_obj = type(getattr(self, obj)).__name__

        if type_obj == 'list' :
            setattr (self, obj, [ _fuzz(x) for x in obj ])
        elif type_obj == 'dict':
            setattr (self, obj, dict([ (_fuzz(k), _fuzz(v)) for k,v in obj.items()]))
        else :
            setattr (self, obj, _fuzz(obj))


def spliturl (netloc, ssl=False):
    """ split hostname and port number (if specified). """

    if netloc in (None,"") :
        return None, None

    try :
        i = netloc.index(":")
        host, port = netloc[:i], int(netloc[i+1:])

    except ValueError :
        host = netloc
        if ssl:
            port = 443
        else :
            port = 80

    return host, port


def called():
    """ @return le nom de la fonction appelee comme une string """
    return stack()[1][3]

def called_line():
    """ @return la ligne de la fonction appelee comme une string """
    return stack()[1][4]

def caller_name():
    """ @return le nom de la fonction appelante comme une string """
    return stack()[2][3]

def caller_line():
    """ @return la ligne de la fonction appelante comme une string """
    return stack()[2][4]


def GET(url, proxy=None):
    """
    Comme la commande Perl LWP GET
    Affiche le code status et message

    @return une liste (header de reponse, contenu de reponse)
    """
    try:
        http = HTTPWrapper(url=url, method="GET", proxy=proxy)
        http.addheader("Accept-Charset", "ISO-8859-1,UTF-8;q=0.7,*;q=0.5")
        http.execute()
        h, b = (http.response_headers, http.response_body)
    except Exception, e:
        ERR("Function %s: %s" % (called(), e))
        h, b = None, None

    return (h,b)


def POST(url, data="", proxy=None):
    """
    Comme la commande Perl POST
    Affiche le code status et message

    @return une liste (header de reponse, contenu de reponse)
    """
    try:
        http = HTTPWrapper(url=url, method="POST", proxy=proxy)
        for o in data.split("&"):
            if len(o) == 0: continue
            k,v = o.split("=",1)
            http.addparam(k,v)

        http.execute()
        h, b = (http.response_headers, http.response_body)
    except Exception,e:
        ERR("%s: %s" % (called(), e))
        h, b = None, None

    return (h,b)


def TRACE(url="/", max_fwd=1, proxy=None):
    """
    Permet de detecter la presence de Reverse Proxy: envoie 2 requetes HTTP
    en changeant le Max_Forward.

    @return une liste avec les headers de 2 HTTPResponse
    """
    h1, h2 = None, None

    try:
        http = HTTPWrapper(url=url, method="TRACE", proxy=proxy)
        http.addheader("Max-Forwards", "0")
        http.execute()
        h1 = http.response_headers
    except Exception,e:
        ERR(e)
        h1 = None

    del(http)
    try:
        http = HTTPWrapper(url=url, method="TRACE", proxy=proxy)
        http.addheader("Max-Forwards", str(max_fwd))
        http.execute()
        h2 = http.response_headers

    except Exception,e:
        ERR(e)
        h2 = None

    return (h1, h2)


def html_encode(chaine, format="html", full=False):
    """
    encode une chaine dans un format donne
    formats actuels supportes :
    - html : html
    - dec : decimal
    - hex : hexadecimal
    pour le decode, cf. html_decode
    """
    res = ""
    if format.lower() not in ("html", "dec", "hex"):
        return res
    for car in chaine :
        if car == '_':
            res += car
            continue
        if not full :
            if '0' <= car <= '9' or \
                    'A' <= car <= 'Z' or\
                    'a' <= car <= 'z' :
                res += car
                continue

        if format == "html" :
            res += "%%%x" % ord(car)
        elif format == "dec" :
            res += "&#%s;" % ord(car)
        elif format == "hex" :
            res += "&#x%x;" % ord(car)

    return res


def html_decode(chaine, format="html"):
    """
    html_decode est l'inverse de la fonction html_encode
    decode une chaine dans un format donne
    formats actuels supportes :
    - html : html
    - dec : decimal
    - hex : hexadecimal
    """
    motifs = {'html':'%', 'dec':'&#', 'hex':'&#x'}
    res = ""
    if format not in motifs.keys() :
        return res

    if format in ('dec', 'hex'):
        chaine = chaine.strip(';')

    motif = motifs[format]
    i = chaine.find(motif)

    if i == -1:
        return chaine
    else:
        res += chaine[:i]
    while i != -1 :

        j = i + len(motif) + 2
        l = chaine[i+len(motif):j]

        if format == "dec":
            letter = int(l)

        elif format in ("html", "hex"):
            letter = int(l, 16)

        res += chr(letter)
        i = chaine[j:].find(motif)
        if i != -1 :
            i += j
            res += chaine[j:i]
        else:
            res += chaine[j:]
    return res


def ascii2mysql(chaine):
    """
    Convertit chaine en forme MySQL CHAR()
    """
    sql = [ "%d" % ord(c) for c in chaine ]
    return "CHAR(" + (",".join(sql)) + ")"


def ascii2mssql (chaine):
    """
    Convertit chaine en forme MS-SQL CHAR()
    """
    sql = [ "CHAR(%d)" % ord(c) for c in chaine ]
    return "+".join(sql)


from urlparse import urlparse

class HTTPReq:
    """
    Classe de parsing de requete HTTP Raw -> Object.
    Permet egalement de retourner cette requete Object -> Raw
    """
    CRLF = "\r\n"
    SEP  = ": "

    def __init__(self, req):
        self.method = 'GET'
        self.path = '/'
        self.version = 'HTTP/1.1'
        self.data = ''
        self.headers = {}

        elts = [ x for x in req.split(self.CRLF) if len(x) ]
        self.method, self.path, self.version = elts.pop(0).split(" ")
        if self.path.startswith("http://") or self.path.startswith("https://"):
            o = urlparse(self.path)
            if len(o.path):
                self.path = o.path
            if len(o.query):
                self.path = o.path + "?" + o.query

        if (req.startswith("POST ")):
            self.data = elts.pop()

        for header in elts:
            try :
                h,v = header.split(self.SEP, 1)
            except ValueError:
                h = header
                v = ''
            finally:
                self.headers[h] = v

    def is_header(self, chunk):
        return self.SEP in chunk

    def __str__(self):
        first = ["%s %s %s" % (self.method, self.path, self.version)]
        headers = ["%s%s%s" % (h,self.SEP,v) for (h,v) in self.headers.items()]

        if self.method == 'POST':
            return self.CRLF.join(first+headers+[self.data])+self.CRLF*2

        else :
            return self.CRLF.join(first+headers)+self.CRLF*2


##################################################
#
#   Some Unicode converting functions
#

def ucs_string(chaine, format=2):

    def ucs_and (a, b):
        i = 0
        while i < len(b):
            a[i] |= b[i]
            i += 1

        res = []
        for i in a :
            res.append('%x' % i)
        return res

    def ucs2(c):
        byte = int('%x' % ord(c), 16)
        parts = []
        parts.append(byte >> 6)   #
        parts.append(byte & 0x3f) # = 0011 1111

        u = []
        u.append(0xC0) # = 1100 0000
        u.append(0x80) # = 1000 0000

        return ucs_and(u, parts)


    def ucs3(c):
        byte = int('%x' % ord(c), 16)

        parts = []
        parts.append(0x00)
        parts.append(byte >> 6)   #
        parts.append(byte & 0x3f) # = 0011 1111

        u = []
        u.append(0xE0) # = 1110 0000
        u.append(0x80) # = 1000 0000
        u.append(0x80) # = 1000 0000

        return ucs_and(u, parts)


    def ucs4(c):
        byte = int('%x' % ord(c), 16)

        parts = []
        parts.append(0x00)
        parts.append(0x00)
        parts.append(byte >> 6)
        parts.append(byte & 0x3f) # = 0011 1111

        u = []
        u.append(0xF0) # = 1111 0000
        u.append(0x80) # = 1000 0000
        u.append(0x80) # = 1000 0000
        u.append(0x80) # = 1000 0000

        return ucs_and(u, parts)


    if format not in xrange(2,5):
        return -1

    res = []

    for c in chaine :
        if format == 2:
            enc = ucs2(c)
        elif format == 3:
            enc = ucs3(c)
        elif format == 4:
            enc = ucs4(c)
        else :
            print ("Wtf ?")
            return -1
        res.extend(enc)

    return res


##################################################
#
# Network functions
#

def expand_cidr(plage, out="/dev/stdout", mode='w'):
    """
    Transforme une plage CIDR en une liste d'IP unique et concatene
    le resultat dans le fichier "out"
    """
    try:
        from netaddr import iter_unique_ips
    except ImportError as ie :
        print("[-] %s" % ie)
        return -1

    regexp = compile("([0-9]{1,3}.){3}[0-9]{1,3}/[0-9]{0,2}")
    if not match(regexp, plage):
        return 0

    with open(out, mode) as fd:
        try:
            for ip in iter_unique_ips(plage):
                fd.write("%s\n" % str(ip))
        except Exception as e:
            print("[-] %s" % e)
            return -1
    return 0


def expand_cidr_file(_file, out="/dev/stdout"):
    """
    Etend en IP unique toutes les plages d'adresses trouvees dans le
    fichier passe en parametre.
    """
    lines = [x.strip() for x in open(_file, 'r').xreadlines()]
    for line in lines :
        expand_cidr(line, out, 'a')
    return 0


def ip2dword(ip):
    b = [int(x) for x in ip.split(".")]
    b.reverse()
    d = 0
    for i in range(len(b)):
        d+= b[i] * (256**i)
    return d


def ip2hex(ip):
    b = ["%#x"%int(x) for x in ip.split(".")]
    return ".".join(b)


def ip2octal(ip):
    b = ["0%0o"%int(x) for x in ip.split(".")]
    return ".".join(b)


##################################################
#
#   Logging functions
#

def now():
    """Return the time of now, suitable for log"""
    return datetime.now().strftime("%d/%m/%y %H:%M:%S")


def OK(msg):
    """ Message display function"""
    stdout.write("[+] %s : %s\n" % (now(), msg))
    stdout.flush()


def INFO(msg):
    """ Information (verbose mode) message display function"""
    stdout.write("[*] %s : %s\n" % (now(), msg))
    stdout.flush()


def ERR(msg):
    """ Error message display function"""
    stderr.write("[!] %s : %s\n" % (now(), msg))
    stderr.flush()


##################################################
#
#   Crypto functions
#

def Levenshtein(s,t):
    """
    implementation de l'algo de distance de Levenshtein
    http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance
    """
    s = ' ' + s
    t = ' ' + t
    d = {}
    S = len(s)
    T = len(t)
    for i in range(S):
        d[i, 0] = i
    for j in range (T):
        d[0, j] = j
    for j in range(1,T):
        for i in range(1,S):
            if s[i] == t[j]:
                d[i, j] = d[i-1, j-1]
            else:
                d[i, j] = min(d[i-1, j] + 1,
                              d[i, j-1] + 1,
                              d[i-1, j-1] + 1)
    return d[S-1, T-1]


def cesar(text, shift):
    '''
    cesar algo implementation
    '''
    cesar = ""

    for c in text:
        if not c.isalpha():
            cesar += c
            continue
        l = ord(c)+shift
        if (c.islower() and l > 122) or (c.isupper() and l > 90):
            l -= 26
        cesar += chr(l)
    return cesar


def vigenere(plaintext, key, decrypt=False):
    '''
    vigenere algo implementation
    '''
    vigenere = ""
    charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"

    if not decrypt: # encrypt
        for i in range(len(plaintext)):
            c = plaintext[i]
            if not c.isalpha():
                vigenere += c
                continue
            c = c.upper()
            k = key[i%len(key)].upper()
            # vigenere += chr(ord(c)^ord(k))
            l = charset[(ord(c)-0x41+ord(k)-0x41)%26]
            vigenere += l

    else:
        for i in range(len(plaintext)):
            c = plaintext[i]
            if not c.isalpha():
                vigenere += c
                continue
            c = c.upper()
            k = key[i%len(key)].upper()
            new_charset = [charset[j%26] for j in range(ord(k)-0x41, ord(k)-0x41+26)]
            vigenere += charset[new_charset.index(c)]

    return vigenere


##################################################
#
# SQL
#

def sqlparse(req, encode=None):
    """
    Parse une requete SQL et la retourne sous forme de dict()
    """
    keywords = ['select', 'limit', 'from', 'where']
    req = req.lower()
    sql_blocks = {}
    new_req = req

    if encode in ("mysql", "MySQL") :
        convert_func = ascii2mysql
    elif encode in ("mssql", "SQLServer") :
        convert_func = ascii2mssql
    else:
        convert_func = str

    # convertion en CHAR()+CHAR()+...
    for i in re.findall("'[^']'", req):
        new_req = re.sub(i, convert_func(i.replace("'","")), new_req)
    req = new_req

    # creation des blocks
    for keyword in keywords:
        if keyword in req:
            block_name = keyword
            block_data = ""
            for word in req[req.find(keyword)+len(keyword):].split(" ") :
                if word in keywords : break
                if word !='' : block_data += " " + word
            sql_blocks[block_name] = block_data.strip()

    return sql_blocks


##################################################
#
#   Misc functions
#


# return an hexdump-style output of src buffer
def hexdump(src, length=0x10):
    f=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
    n=0
    result=''

    while src:
       s,src = src[:length],src[length:]
       hexa = ' '.join(["%02X"%ord(x) for x in s])
       s = s.translate(f)
       result += "%04X   %-*s   %s\n" % (n, length*3, hexa, s)
       n+=length

    return result


def xor_crypt_string(data, key):
    return ''.join(chr(ord(x) ^ ord(y)) for (x,y) in izip(data, cycle(key)))