Example #1
0
 def _grep(self, request, response):
     # The grep process is all done in another thread. This improves the
     # speed of all w3af.
     url_instance = request.url_object
     domain = url_instance.getDomain()
     
     if self._grepPlugins and domain in cf.cf.getData('targetDomains'):
         
         # I'll create a fuzzable request based on the urllib2 request object
         fr = create_fuzzable_request(
                                     url_instance,
                                     request.get_method(),
                                     request.get_data(),
                                     request.headers
                                     )
         
         msg = 'Starting "grep_worker" for response: "%s"' % repr(response)
         om.out.debug( msg )
         
         for grep_plugin in self._grepPlugins:
             #
             #   For debugging, do not remove, only comment out if needed.
             #
             self._grep_worker( grep_plugin, fr, response )
             
             # TODO: Analyze if creating a different threadpool for grep workers speeds 
             #       up the whole process
             #targs = (grep_plugin, fuzzReq, response)
             #self._tm.startFunction( target=self._grep_worker, args=targs, 
             #                        ownerObj=self, restrict=False )
         
         msg = 'Finished "grep_worker" for response: "%s"' % repr(response)
         om.out.debug( msg )
         
         self._tm.join( self )
Example #2
0
 def log_req_resp(request, response):
     '''
     Send the request and the response to the output manager.
     '''        
     fr = create_fuzzable_request(
                          request, add_headers=request.unredirected_hdrs)
     if isinstance(response, httpResponse.httpResponse):
         resp = response
     else:
         resp = httpResponse.from_httplib_resp(
                               response, original_url=request.url_object)
         resp.setId(response.id)
     
     om.out.logHttp(fr, resp)
Example #3
0
    def store_in_cache(request, response):
        hi = HistoryItem()

        # Set the request
        req = create_fuzzable_request(request, add_headers=request.unredirected_hdrs)
        hi.request = req

        # Set the response
        resp = httpResponse.from_httplib_resp(response, original_url=request.url_object)
        resp.setId(response.id)
        resp.setAlias(gen_hash(request))
        hi.response = resp

        # Now save them
        try:
            hi.save()
        except KeyboardInterrupt, k:
            raise k
Example #4
0
 def mangleRequest(self, request):
     '''
     This method mangles the request.
     
     @param request: This is the request to mangle.
     @return: A mangled version of the request.
     '''
     data = request.getData()
     for regex, string in self._req_body_manglers:
         data = regex.sub(string, data)
     
     header_string = headersToString(request.getHeaders())
     for regex, string in self._req_head_manglers:
         header_string = regex.sub(string, header_string)
     header_dict = stringToHeaders(header_string)
     
     return create_fuzzable_request(
                              request.getURL(),
                              request.getMethod(),
                              data, header_dict
                              )
Example #5
0
    def _objs_from_log( self, req_file ):
        '''
        This code was largely copied from Bernardo Damele's sqlmap[0] . See
        __feedTargetsDict() in lib/core/options.py. So credits belong to the
        sqlmap project.

        [0] http://sqlmap.sourceforge.net/

        @author Patrick Hof
        '''
        res = []
        fp = open( req_file, "r" )
        fread = fp.read()
        fread = fread.replace( "\r", "" )
        req_res_list = fread.split( "======================================================" )
        
        port   = None
        scheme = None

        for request in req_res_list:
            if scheme is None:
                scheme_port = re.search(
                        "\d\d[\:|\.]\d\d[\:|\.]\d\d\s+(http[\w]*)\:\/\/.*?\:([\d]+)",
                        request,
                        re.I
                )

            if scheme_port:
                scheme = scheme_port.group( 1 )
                port   = scheme_port.group( 2 )

            if not re.search ( "^[\n]*(GET|POST).*?\sHTTP\/", request, re.I ):
                continue

            if re.search( "^[\n]*(GET|POST).*?\.(gif|jpg|png)\sHTTP\/", request, re.I ):
                continue

            method       = None
            url          = None
            postdata     = None
            host = None
            headers      = {}
            get_post_req = False
            lines        = request.split( "\n" )

            for line in lines:
                if len( line ) == 0 or line == "\n":
                    continue

                if line.startswith( "GET " ) or line.startswith( "POST " ):
                    if line.startswith( "GET " ):
                        index = 4
                    else:
                        index = 5

                    url    = line[index:line.index(" HTTP/")]
                    method = line[:index-1]

                    get_post_req = True

                # XXX do we really need this? This is from the sqlmap code.
                # 'data' would be 'postdata' here. I can't figure out why this
                # is needed. Does WebScarab occasionally split requests to a new
                # line if they are overly long, so that we need to search for
                # GET parameters even after the URL was parsed? But that
                # wouldn't make sense with the way 'url' is set in line 168.
                # 
                # GET parameters 
                # elif "?" in line and "=" in line and ": " not in line:
                #     data    = line

                # Parse headers
                elif ": " in line:
                    key, value = line.split(": ", 1)
                    headers[key] = value
                    
                    if key.lower() == 'host':
                        host = value

                # POST parameters
                elif method is not None and method == "POST" and "=" in line:
                    postdata = line

            if get_post_req:
                if not url.startswith( "http" ):
                    url    = "%s://%s:%s%s" % ( scheme or "http", host, port or "80", url )
                    scheme = None
                    port   = None

                url_instance = url_object(url)
                res.append(
                       create_fuzzable_request(
                                  url_instance, method,
                                  postdata, headers
                                  )
                           )
                
        return res
Example #6
0
        >>> pdr._dc
        QueryString({u'id': [u'1']})
        >>> pdr.getData()
        'id=1'
        '''
        try:
            (method, uri, postdata) = csv_row
        except ValueError, value_error:
            msg = 'The file format is incorrect, an error was found while parsing: "'
            msg += str(csv_row) + '". Exception: "' + str(value_error) + '".'
            om.out.error(msg)
        else:
            # Create the obj based on the information
            uri = url_object(uri)
            if uri.is_valid_domain():
                return create_fuzzable_request(uri, method, postdata)

    def _objs_from_log( self, req_file ):
        '''
        This code was largely copied from Bernardo Damele's sqlmap[0] . See
        __feedTargetsDict() in lib/core/options.py. So credits belong to the
        sqlmap project.

        [0] http://sqlmap.sourceforge.net/

        @author Patrick Hof
        '''
        res = []
        fp = open( req_file, "r" )
        fread = fp.read()
        fread = fread.replace( "\r", "" )
Example #7
0
def httpRequestParser(head, postdata):
    '''
    This function parses HTTP Requests from a string to a fuzzableRequest.
    
    @parameter head: The head of the request.
    @parameter postdata: The post data of the request
    @return: A fuzzableRequest object with all the corresponding information
        that was sent in head and postdata
    
    @author: Andres Riancho ( [email protected] )

    >>> httpRequestParser('200 http://www.w3af.com/ HTTP/1.0', 'foo=bar')
    <postdata fuzzable request | 200 | http://www.w3af.com/>

    >>> httpRequestParser('200 http://www.w3af.com/ HTTP/1.0', '')
    <QS fuzzable request | 200 | http://www.w3af.com/>

    >>> httpRequestParser('200 / HTTP/1.0', '')
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    w3afException: You have to specify the complete URI, including the protocol and the host. Invalid URI: /

    >>> httpRequestParser('ABCDEF', '')
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    w3afException: The HTTP request has an invalid <method> <uri> <version> token: "ABCDEF".

    >>> head = "200 http://www.w3af.com/ HTTP/1.0"
    >>> head += '\\nHost: www.w3af.com'
    >>> httpRequestParser( head, 'foo=bar')
    <postdata fuzzable request | 200 | http://www.w3af.com/>
    '''
    # Parse the request head
    splitted_head = head.split('\n')
    splitted_head = [h.strip() for h in splitted_head if h]
    
    if not splitted_head:
        msg = 'The HTTP request is invalid.'
        raise w3afException(msg)        
    
    # Get method, uri, version
    metUriVer = splitted_head[0]
    firstLine = metUriVer.split(' ')
    if len(firstLine) == 3:
        # Ok, we have something like "GET /foo HTTP/1.0". This is the best case for us!
        method, uri, version = firstLine
    elif len(firstLine) < 3:
        msg = 'The HTTP request has an invalid <method> <uri> <version> token: "'
        msg += metUriVer +'".'
        raise w3afException(msg)
    elif len(firstLine) > 3:
        # GET /hello world.html HTTP/1.0
        # Mostly because we are permissive... we are going to try to send the request...
        method = firstLine[0]
        version = firstLine[-1]
        uri = ' '.join( firstLine[1:-1] )
    
    checkVersionSyntax(version)
    
    # If we got here, we have a nice method, uri, version first line
    # Now we parse the headers (easy!) and finally we send the request
    headers = splitted_head[1:]
    headersDict = {}
    for header in headers:
        one_splitted_header = header.split(':', 1)
        if len(one_splitted_header) == 1:
            raise w3afException('The HTTP request has an invalid header: "' + header + '"')
        headersDict[ one_splitted_header[0].strip() ] = one_splitted_header[1].strip()
    host = ''
    for headerName in headersDict:
        if headerName.lower() == 'host':
            host = headersDict[headerName]
    uri = url_object(checkURISyntax(uri, host))
    return create_fuzzable_request(uri, method, postdata, headersDict)