예제 #1
0
    def _on_request(self,
                    urlopen,
                    path,
                    pool,
                    method,
                    url,
                    body=None,
                    headers=None,
                    **kw):
        # Remove bypass headers
        real_headers = dict(headers or {})
        real_headers.pop(URLLIB3_BYPASS)

        # Create request contract based on incoming params
        req = Request(method)
        req.headers = real_headers
        req.body = body

        # Compose URL
        req.url = '{}://{}:{:d}{}'.format(pool.scheme, pool.host, pool.port
                                          or 80, url)

        # Match the request against the registered mocks in pook
        mock = self.engine.match(req)

        # If cannot match any mock, run real HTTP request since networking
        # or silent model will be enabled, otherwise this statement won't
        # be reached (an exception will be raised before).
        if not mock:
            return urlopen(pool, method, url, body=body, headers=headers, **kw)

        # Shortcut to mock response and response body
        res = mock._response
        body = res._body

        # Aggregate headers as list of tuples for interface compatibility
        headers = []
        for key in res._headers:
            headers.append((key, res._headers[key]))

        if is_chunked_response(headers):
            body_chunks = body if isinstance(body, list) else [body]
            body_chunks = [chunk.encode() for chunk in body_chunks]

            body = ClientHTTPResponse(MockSock)
            body.fp = FakeChunkedResponseBody(body_chunks)
        else:
            # Assume that the body is a bytes-like object
            body = body_io(body)

        # Return mocked HTTP response
        return HTTPResponse(
            path,
            body=body,
            status=res._status,
            headers=headers,
            preload_content=False,
            reason=http_reasons.get(res._status),
            original_response=FakeResponse(method, headers),
        )
예제 #2
0
    def testPipelining(self):
        # Tests the use of several requests issued at once.
        s = ("GET / HTTP/1.0\r\n"
             "Connection: %s\r\n"
             "Content-Length: %d\r\n"
             "\r\n"
             "%s")
        to_send = ''
        count = 25
        for n in range(count):
            body = "Response #%d\r\n" % (n + 1)
            if n + 1 < count:
                conn = 'keep-alive'
            else:
                conn = 'close'
            to_send += s % (conn, len(body), body)

        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(to_send)
        for n in range(count):
            expect_body = "Response #%d\r\n" % (n + 1)
            response = ClientHTTPResponse(sock)
            response.begin()
            self.failUnlessEqual(int(response.status), 200)
            length = int(response.getheader('Content-Length', '0'))
            response_body = response.read(length)
            self.failUnlessEqual(length, len(response_body))
            self.failUnlessEqual(response_body, expect_body)
예제 #3
0
 def proxy(self):
     if self.remote is None or self.lastHost != self.headers['Host']:
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.remote.connect((proxy_host, proxy_port))
     self.remote.sendall(self.requestline.encode('ascii') + b'\r\n')
     # Add auth inf.
     self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(auth)
     #self.remote.sendall('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(auth))
     headerstr = str(self.headers).replace('\r\n', '\n').replace('\n', '\r\n')
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
     # Send Post data
     if self.command == 'POST':
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
     response = HTTPResponse(self.remote, method=self.command)
     response.begin()
     
     # Reply to the browser
     status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
     self.wfile.write(status.encode('ascii') + b'\r\n')
     hlist = []
     for line in response.msg.headers: # Fixed multiple values of a same name
         if 'TRANSFER-ENCODING' not in line.upper():
             hlist.append(line)
     self.wfile.write(''.join(hlist) + b'\r\n')
     
     if self.command == 'CONNECT' and response.status == 200:
         return self.transfer(self.remote, self.connection)
     else:
         while True:
             response_data = response.read(BufferSize)
             if not response_data: break
             self.wfile.write(response_data)
예제 #4
0
 def parseNewResponse(self, packetString, src, dst, tcp):
     responseCode = packetString[9:12]
     if responseCode == '200': # just okay responses for now        
         if '\r\n\r\n' in packetString: # only proceed if the response has a body
             bodyIndex = packetString.index('\r\n\r\n') + 4
             body = packetString[bodyIndex:]
             socket = FakeSocket(packetString)
             response = HTTPResponse(socket)
             response.begin()
             headerArray = response.getheaders()
             for item in headerArray:
                 flowKey = (src, dst)
                 if item[0] == 'content-type' and 'text/html' in item[1]: # accept any kind of text content
                     print headerArray
                     for item in headerArray:
                         if item[0] == 'content-length':
                             print 'found fixed length'
                             length = int(item[1])
                             if length is not 0:
                                 self.parseFixedLengthResponse(flowKey, body, length, src, dst, tcp, responseCode)
                             else:
                                 print "warning, content-length is zero!"
                         elif item[0] == 'transfer-encoding' and item[1] == 'chunked':
                             print 'found chunked'
                             self.parseChunkedResponse(flowKey, body, src, dst, tcp, responseCode)
         else:
             print "body not found"
    def sogouProxy(self):
        if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'):  #Give a PAC file
            self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n')
            hstr = '''Host: 127.0.0.1

function FindProxyForURL(url, host) {
  if (url.substring(0,5) == 'http:' && 
      !isPlainHostName(host) && 
      !shExpMatch(host, '*.local') && 
      !isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') && 
      !isInNet(dnsResolve(host), '172.16.0.0',  '255.240.0.0') && 
      !isInNet(dnsResolve(host), '192.168.0.0',  '255.255.0.0') && 
      !isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') ) 
    return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT';
  return 'DIRECT';
}'''
            self.wfile.write(hstr + b'\r\n')
            return
            
        if self.remote is None or self.lastHost != self.headers["Host"]:
            if PROXY_MODE == 'HTTPS':
                context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                context.verify_mode = ssl.CERT_REQUIRED
                context.check_hostname = True
                context.load_default_certs()
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.settimeout(RemoteTimeout)
                self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net')
                self.remote.connect(('proxy.googlezip.net', 443))
            else:  #HTTP
                self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                self.remote.settimeout(RemoteTimeout)
                self.remote.connect(("compress.googlezip.net", 80))
        self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
        # Add Verification Tags
        self.headers["Chrome-Proxy"] = get_google_header()
        headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
        self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
        # Send Post data
        if self.command == 'POST':
            self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
        response = HTTPResponse(self.remote, method=self.command)
        response.begin()

        # Reply to the browser
        status = "HTTP/1.1 " + str(response.status) + " " + response.reason
        self.wfile.write(status.encode('ascii') + b'\r\n')
        hlist = []
        for line in response.msg.headers: # Fixed multiple values of a same name
            if 'TRANSFER-ENCODING' not in line.upper():
                hlist.append(line)
        self.wfile.write("".join(hlist) + b'\r\n')

        if self.command == "CONNECT":  # NO HTTPS, as Chrome DCP does not allow HTTPS traffic
            return
        else:
            while True:
                response_data = response.read(BufferSize)
                if not response_data: break
                self.wfile.write(response_data)
예제 #6
0
 def get_json(self, response):
     res_str = str(helpers.bytesToString(response.getResponse()))
     source = FakeSocket(res_str)
     res_obj = HTTPResponse(source)
     res_obj.begin()
     print "Reading response object"
     return json.loads(res_obj.read())
예제 #7
0
파일: http.py 프로젝트: 1stvamp/apiary
    def send_request(self, request):
        self.request_num += 1

        # Sanity check: if we're sending a request with a content-length but
        # we don't have that many bytes to send, we'll just get a 504.  Don't
        # send it and instead report a client error.

        parts = request.split('\r\n\r\n', 1)

        if len(parts) > 1:
            req, body = parts

            match = content_length_re.search(req)
            if match:
                if len(body) < int(match.group(1)):
                    self.error("request body of incorrect size")

                    return True

        if not self.connection:
            self._connect()

        if self.connection:
            # tally request method
            #self.tally(request.split(" ", 1)[0])

            try:
                self.connection.sendall(request)

                response = HTTPResponse(self.connection)
                response.begin()

                self.tally(response.status)

                while response.read():
                    pass

                if response.will_close:
                    # We hope our Connection: keep-alive won't be ignored, but
                    # deal with it if it does.
                    self._disconnect()

                if self.options.speedup < 0.8:
                    # if we're slowing down by too much, keep-alive will just
                    # result in the server getting bored between requests and
                    # dropping the connection, so disable it.
                    self._disconnect()

                return True
            except IncompleteRead:
                self.error("error while reading response: IncompleteRead (terminating job)")
                self._disconnect()

            except Exception, e:  # TODO: more restrictive error catching?
                self.error("error while sending request and reading response: %s %s" % (type(e), e))
                self._disconnect()

                if self.connection:
                    self.connection.close()
                    self.connection = None
예제 #8
0
 def do_proxy(self):
     try:
         if self.s == 0:
             self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self.s.connect((proxy_host, proxy_port))
         self.s.send(self.requestline.encode('ascii') + b'\r\n')
         # Add Sogou Verification Tags
         self.headers['X-Sogou-Auth'] = x_sogou_auth
         t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
         self.headers['X-Sogou-Tag'] = calc_sogou_hash(t, self.headers['Host'])
         self.headers['X-Sogou-Timestamp'] = t
         self.s.send(str(self.headers).encode('ascii') + b'\r\n')
         # Send Post data
         if(self.command=='POST'):
             self.s.send(self.rfile.read(int(self.headers['Content-Length'])))
         response = HTTPResponse(self.s, method=self.command, buffering=True)
         response.begin()
         # Reply to the browser
         status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
         self.wfile.write(status.encode('ascii') + b'\r\n')
         h = ''
         for hh, vv in response.getheaders():
             if hh.upper()!='TRANSFER-ENCODING':
                 h += hh + ': ' + vv + '\r\n'
         self.wfile.write(h.encode('ascii') + b'\r\n')
         while True:
             response_data = response.read(8192)
             if(len(response_data) == 0):
                 break
             self.wfile.write(response_data)
     except socket.error:
         print('socket error for ' + self.requestline)
예제 #9
0
파일: proxy.py 프로젝트: beelives/NPUcat
 def sogouProxy(self):
     if self.remote is None or self.lastHost != self.headers["Host"]:
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.remote.settimeout(RemoteTimeout)
         self.remote.connect((proxy_host, proxy_port))
     self.remote.sendall(self.requestline.encode('ascii').replace('baike.baidu.com', '220.181.111.247') + b"\r\n")
     # Add Sogou Verification Tags
     self.headers["X-Sogou-Auth"] = x_sogou_auth
     t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
     self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
     self.headers["X-Sogou-Timestamp"] = t
     headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
     # Send Post data
     if self.command == 'POST':
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
     response = HTTPResponse(self.remote, method=self.command)
     response.begin()
     
     # Reply to the browser
     status = "HTTP/1.1 " + str(response.status) + " " + response.reason
     self.wfile.write(status.encode('ascii') + b'\r\n')
     hlist = []
     for line in response.msg.headers: # Fixed multiple values of a same name
         if 'TRANSFER-ENCODING' not in line.upper():
             hlist.append(line)
     self.wfile.write("".join(hlist) + b'\r\n')
     
     if self.command == "CONNECT" and response.status == 200:
         return self.transfer(self.remote, self.connection)
     else:
         while True:
             response_data = response.read(BufferSize)
             if not response_data: break
             self.wfile.write(response_data)
예제 #10
0
 def sogouProxy(self):  
     if self.remote is None or self.lastHost != self.headers["Host"]:  
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  
         self.remote.settimeout(RemoteTimeout)  
         self.remote.connect((proxy_host, proxy_port))  
     self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")  
     # Add Sogou Verification Tags  
     self.headers["X-Sogou-Auth"] = x_sogou_auth  
     t = hex(int(time.time()))[2:].rstrip('L').zfill(8)  
     self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])  
     self.headers["X-Sogou-Timestamp"] = t  
     headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")  
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")  
     # Send Post data  
     if self.command == 'POST':  
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))  
     response = HTTPResponse(self.remote, method=self.command)  
     response.begin()  
       
     # Reply to the browser  
     status = "HTTP/1.1 " + str(response.status) + " " + response.reason  
     self.wfile.write(status.encode('ascii') + b'\r\n')  
     hlist = []  
     for line in response.msg.headers: # Fixed multiple values of a same name  
         if 'TRANSFER-ENCODING' not in line.upper():  
             hlist.append(line)  
     self.wfile.write("".join(hlist) + b'\r\n')  
       
     if self.command == "CONNECT" and response.status == 200:  
         return self.transfer(self.remote, self.connection)  
     else:  
         while True:  
             response_data = response.read(BufferSize)  
             if not response_data: break  
             self.wfile.write(response_data)  
예제 #11
0
    def testPipelining(self):
        # Tests the use of several requests issued at once.
        s = ("GET / HTTP/1.0\r\n"
             "Connection: %s\r\n"
             "Content-Length: %d\r\n"
             "\r\n"
             "%s")
        to_send = ''
        count = 25
        for n in range(count):
            body = "Response #%d\r\n" % (n + 1)
            if n + 1 < count:
                conn = 'keep-alive'
            else:
                conn = 'close'
            to_send += s % (conn, len(body), body)

        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(to_send)
        for n in range(count):
            expect_body = "Response #%d\r\n" % (n + 1)
            response = ClientHTTPResponse(sock)
            response.begin()
            self.failUnlessEqual(int(response.status), 200)
            length = int(response.getheader('Content-Length', '0'))
            response_body = response.read(length)
            self.failUnlessEqual(length, len(response_body))
            self.failUnlessEqual(response_body, expect_body)
예제 #12
0
def issue_row(raw_row):
    issue_row = {}
    for column in COLUMN_HEADERS:
        column_data_raw = raw_row.findtext(column)
        if column_data_raw:
            if column in ['issueDetail', 'issueBackground', 'remediationBackground']:
                issue_row[column] = htmltext(column_data_raw)
            else:
                issue_row[column] = column_data_raw

            if len(issue_row[column]) > 32000:
                issue_row[column] = "".join(issue_row[column][:32000], " [Text Cut Due To Length]")

    request = raw_row.findtext('./requestresponse/request')
    if request:
        parsed_request = HTTPRequest(binascii.a2b_base64(request))
        formatted_request_a = "command : {}\nuri : {}\nrequest_version : {}".format(parsed_request.command, parsed_request.path, parsed_request.request_version)
        formatted_request_b = "\n".join("{}: {}".format(header, parsed_request.headers[header]) for header in parsed_request.headers.keys())
        issue_row['requestHeaders'] = "{}\n{}".format(formatted_request_a, formatted_request_b)

    response = raw_row.findtext('./requestresponse/response')
    if response:
        parsed_response = HTTPResponse(FakeSocket(binascii.a2b_base64(response)))
        parsed_response.begin()
        formatted_response = "\n".join(["{} : {}".format(header_item[0], header_item[1]) for header_item in parsed_response.getheaders()])
        issue_row['responseHeaders'] = formatted_response

    return issue_row
예제 #13
0
def open_image2(url, buff_size=4096, connect_timeout=5, read_timeout=4, 
        full_read_timeout=30, logger=logging):
    t = urlparse.urlparse(url)
    path = t.path or '/'
    if len(t.netloc.split(':')) == 2:
        HOST, PORT = t.netloc.split(':')
    else:
        HOST, PORT = t.netloc, 80
    UA = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36'
            ' (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36')
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(connect_timeout)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    try:
        s.connect((HOST, PORT))
    except Exception as e:
        logger.exception(e)
        return None
    s.send("GET %s HTTP/1.1\r\n" % path)
    s.send("Host:%s\r\n" % HOST)
    s.send("Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n")
    s.send("User-Agent:%s\r\n" % UA)
    s.send('\r\n')
    data = []
    begin = time.time()
    while 1:
        if time.time() - begin > full_read_timeout:
            logger.error('reach full_read_timeout')
            data = []
            break
        try:
            msg = s.recv(buff_size)
            #print len(msg), len(data), time.time() - begin
            if len(msg) == 0:
                break
            data.append(msg)
        except socket.timeout as e:
            if e.args[0] == 'timed out':
                break
            else:
                logger.error('open image [%s] socket timeout: %s' % (url, str(e)))
                data = []
                break
        except socket.error as e:
            logger.error('open image [%s] socket error: %s' % (url, str(e)))
            data = []
            break
    s.shutdown(1)
    s.close()
    
    data = ''.join(data)
    if data:
        resp = HTTPResponse(FakeSocket(data))
        resp.begin()
        if resp.status == 200:
            return resp.read(len(data))
    return None
예제 #14
0
 def __init__(self, response):
     r = HTTPResponse(self._FakeSocket(response))
     r.begin()
     self.location = r.getheader("location")
     self.usn      = r.getheader("usn")
     self.st       = r.getheader("st")
     self.cache    = r.getheader("cache-control").split("=")[1]
     try: r.close()
     except: pass
예제 #15
0
def httpparse(fp):
    try:
        contents = fp.read()
    except AttributeError:
        contents = fp
    socket = FakeSocket(contents)
    response = HTTPResponse(socket)
    response.begin()
    return response
예제 #16
0
    def get(self, request):

        # Format and pass along the initial request
        request = self.format_request(request)
        self.outsocket.sendall(request)

        # Wait for request
        while (True):
            r, w, x = select.select([self.insocket, self.outsocket], [], [], 0)
            if self.insocket in r:
                request = self.insocket.recv(buff_size)
                if request == "":
                    print "Got end message from browser"
                    self.kill()
                else:
                    try:
                        request = self.format_request(request)
                    except MismatchedHost:
                        print "Host changed. Getting new end socket"
                        self.insocket.sendall("")
                        self.kill()
                self.outsocket.sendall(request)
            if self.outsocket in r:
                httpRes = HTTPResponse(self.outsocket)
                response = ""
                try:
                    httpRes.begin()

                    headers = str(httpRes.msg)
                    print headers
                    content = httpRes.read()

                    # TODO: Move below to format_response
                    # Fix chunked header
                    if headers.find("Transfer-Encoding: chunked") != -1:
                        headers = headers.replace(
                            "Transfer-Encoding: chunked\r\n", "")
                        headers += "Content Length: " + str(
                            len(content)) + "\r\n"

                    if httpRes.version == 10:
                        response += "HTTP/1.0 "
                    elif httpRes.version == 11:
                        response += "HTTP/1.1 "
                    response += str(httpRes.status) + " " + str(
                        httpRes.reason) + nl
                    response += headers + nl
                    response += content
                    print response
                except BadStatusLine:
                    self.kill()
                self.insocket.sendall(response)
                self.kill()
            if self.killed:
                self.end()
                return
예제 #17
0
    def init(self, response_text):
        class FakeSocket:
            def __init__(self, response_str):
                self._file = StringIO(response_str)
            def makefile(self, *args, **kwargs):
                return self._file

        source = FakeSocket(response_text)
        HTTPResponse.__init__(self, source)
        self.begin()
예제 #18
0
def filter_malware(response_str):
    global apikey
    source = HTTPObject(response_str)
    http_response = HTTPResponse(source)
    http_response.begin()
    url = 'https://www.virustotal.com/vtapi/v2/file/report'
    content = http_response.read(len(response_str))
    hash = hashlib.md5(content).hexdigest()
    params = {'apikey': apikey, 'resource': hash}
    response = requests.get(url, params=params)
    return response
예제 #19
0
    def get_response(self):
        """Return the parse XML response.

        If no data was read, raises EOFError."""
        self.read_buffer.seek(0)
        if self.amount_read == 0:
            raise EOFError("Read zero bytes")
        self.logger.debug("Processing response of %d bytes" % self.amount_read)
        response = HTTPResponse(self.read_buffer)
        response.begin()  # Process the response
        return response
예제 #20
0
 def _parse_http(self, text):
     # if the response text starts with a 302, skip to the next non-302 header
     if re.match(r'^HTTP/.*?\s302 Found', text):
         m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
         if not m:
             raise Exception("Unrecognized response: %s" % text)
         else:
             text = m.group(1)
     socket = self.FakeSocket(text)
     response = HTTPResponse(socket)
     response.begin()
     return response
예제 #21
0
def parse_response_header(seg):
    source = FakeSocket(seg)
    response = HTTPResponse(source)
    response.begin()
    http_v = {10: 'HTTP/1.0', 11: 'HTTP/1.1'}
    res_data = {}
    res_data['status'] = response.status
    res_data['response_version'] = http_v[response.version]
    data = response.getheaders()
    for h in data:
        res_data[str(h[0])] = h[1]
    return res_data
예제 #22
0
 def _parse_http(self, text):
     # if the response text starts with a 302, skip to the next non-302 header
     if re.match(r'^HTTP/.*?\s302 Found', text):
         m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
         if not m:
             raise Exception("Unrecognized response: %s" % text)
         else:
             text = m.group(1)
     socket = self.FakeSocket(text)
     response = HTTPResponse(socket)
     response.begin()
     return response
예제 #23
0
 def proccessHttpResponse(self, burpHttpReqResp):
     """ Processes only the HTTP repsonses with a CSP header """
     byteResponse = burpHttpReqResp.getResponse()
     httpSocket = HttpDummySocket(bytearray(byteResponse))
     response = HTTPResponse(httpSocket)
     response.begin()
     issues = []
     for header in response.getheaders():
         if header[0].lower() in ContentSecurityPolicy.HEADERS:
             findings = self.parseContentSecurityPolicy(header, burpHttpReqResp)
             issues.extend(findings)
     return issues
예제 #24
0
 def proccessHttpResponse(self, burpHttpReqResp):
     """ Processes only the HTTP repsonses with a CSP header """
     byteResponse = burpHttpReqResp.getResponse()
     httpSocket = HttpDummySocket(bytearray(byteResponse))
     response = HTTPResponse(httpSocket)
     response.begin()
     issues = []
     for header in response.getheaders():
         if header[0].lower() in ContentSecurityPolicy.HEADERS:
             findings = self.parseContentSecurityPolicy(header, burpHttpReqResp)
             issues.extend(findings)
     return issues
예제 #25
0
def decompress_payload(payload):
    try:
        source = FakeSocket(payload)
        res = HTTPResponse(source)
        res.begin()
        result = zlib.decompress(res.read(), 16 + zlib.MAX_WBITS)
    except Exception as e:
        result = payload
        # try:
        #     result = '.'.join(str(ord(c)) for c in payload)
        # except:
        #     result = payload
    return result
예제 #26
0
    def begin(self):
        HTTPResponse.begin(self)
        # To void the broken logic at the end of begin, because
        # but the connection may stay explicitly open with a connect
        # of for some other reason
        self.will_close = self._check_close()

        # it is sensible to assume that after a connect if
        # 200 is returned, the connection will not close
        # even if issued as 1.0 and no specific connection
        # header came back - a binary connection is now open
        if self._method == 'CONNECT' and self.status == 200:
            self.will_close = 0
예제 #27
0
def getBody(msg):
    ''' Get the entity body section of the http message '''
    class FakeSocket():
        def __init__(self, msg):
            self.file = StringIO(msg)

        def makefile(self, *args, **kwargs):
            return self.file

    source = FakeSocket(msg)
    rsp = HTTPResponse(source)
    rsp.begin()
    return rsp.read(len(msg))
예제 #28
0
    def begin(self):
        HTTPResponse.begin(self)
        # To void the broken logic at the end of begin, because
        # but the connection may stay explicitly open with a connect
        # of for some other reason
        self.will_close = self._check_close()

        # it is sensible to assume that after a connect if
        # 200 is returned, the connection will not close
        # even if issued as 1.0 and no specific connection
        # header came back - a binary connection is now open
        if self._method == 'CONNECT' and self.status == 200:
            self.will_close = 0
    def setUp(self):
        from httplib import HTTPResponse

        with Stub() as HTTPResponse:
            HTTPResponse.status >> 200
            HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "******", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}'

        with Stub() as conn:
            from httplib import HTTPConnection

            conn = HTTPConnection("api.junar.com")
            conn.request("POST", "/datastreams/publish", self.params, self.headers)
            conn.getresponse() >> HTTPResponse
예제 #30
0
    def get(self, request):
 
        # Format and pass along the initial request
        request = self.format_request(request)
        self.outsocket.sendall(request)
 
        # Wait for request
        while(True):
            r,w,x = select.select([self.insocket, self.outsocket],[],[],0)
            if self.insocket in r:
                request = self.insocket.recv(buff_size)
                if request == "":
                    print "Got end message from browser"
                    self.kill()
                else:
                    try:
                        request = self.format_request(request)
                    except MismatchedHost:
                        print "Host changed. Getting new end socket"
                        self.insocket.sendall("")
                        self.kill()                
                self.outsocket.sendall(request)
            if self.outsocket in r: 
                httpRes = HTTPResponse(self.outsocket)
                response = ""
                try:
                    httpRes.begin()
                    
                    headers = str(httpRes.msg)
                    content = httpRes.read()
                    
                    if headers.find("Transfer-Encoding: chunked") != -1:
                        headers = headers.replace("Transfer-Encoding: chunked\r\n", "")
                        headers += "Content Length: " + str(len(content)) + "\r\n"
                    
                    if httpRes.version == 10:
                        response += "HTTP/1.0 "
                    elif httpRes.version == 11:
                        response += "HTTP/1.1 "
                    response += str(httpRes.status) + " " + str(httpRes.reason) + nl
                    response += headers + nl
                    response += content
                    #print response
                except BadStatusLine:
                    self.kill()
                self.insocket.sendall(response)
                self.kill()
                #'''
            if self.killed:
                self.end()
                return           
예제 #31
0
    def send_and_recv(self):
        try:
            # because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
            # send message to www.dream-pro.info with socket.socket to make
            # http connection
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((WWW_DREAM_PRO_INFO, 80))
            sock.sendall(str(self))
        except:
            print 'SocketError'
            return

        res = HTTPResponse(sock)
        res.begin()
        res_body = res.read()
        res.close()

        if 'transfer-encoding' in res.msg:
            # httplib.HTTPResponse automatically concatenate chunked response
            # but do not delete 'transfer-encoding' header
            # so the header must be deleted
            res.msg.__delitem__('transfer-encoding')
        compmeth = res.msg.getheader('content-encoding', '').lower()
        if compmeth and compmeth.find('identity') != 0:
            # response body is compressed with some method
            offset = 0
            if compmeth.find('gzip') != -1:
                # if body is gziped, header offset value is 47
                # if not, offset value is 0
                # this server does not support sdch...
                offset += 47
            res_body = decompress(res_body, offset)
            res.msg['content-encoding'] = 'identity'

        return res, res_body
예제 #32
0
    def testKeepaliveHttp10(self):
        # Handling of Keep-Alive within HTTP 1.0
        data = "Default: Don't keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        # We sent no Connection: Keep-Alive header
        # Connection: close (or no header) is default.
        self.failUnless(connection != 'Keep-Alive')

        # If header Connection: Keep-Alive is explicitly sent,
        # we want to keept the connection open, we also need to return
        # the corresponding header
        data = "Keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Connection: Keep-Alive\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        self.failUnlessEqual(connection, 'Keep-Alive')
예제 #33
0
    def sendAndReceive(self):
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((ORIGINAL_IP, 80))
            sock.sendall(str(self))
        except:
            return GlobalTools.FailedHTTPResponse(), ''

        res = HTTPResponse(sock)
        res.begin()
        res_body = res.read()
        res.close()

        if 'transfer-encoding' in res.msg:
            # httplib.HTTPResponse automatically concatenate chunked response
            # but do not delete 'transfer-encoding' header
            # so the header must be deleted
            res.msg.__delitem__('transfer-encoding')
        compmeth = res.msg.getheader('content-encoding', '').lower()
        if compmeth and compmeth.find('identity') != 0:
            # response body is compressed with some method
            offset = 0
            if compmeth.find('gzip') != -1:
                # if body is gziped, header offset value is 47
                # if not, offset value is 0
                # this server does not support sdch...
                offset += 47
            res_body = decompress(res_body, offset)
            res.msg['content-encoding'] = 'identity'

        return res, res_body
예제 #34
0
	def validate_for_regular_http_api_response(self,request_verb):
		try:
			s = self.get_socket(self.host,self.current_port)
			request_string = self.get_request_templates(request_verb)
			#print request_string 
			s.sendall(request_string)
			data = HTTPResponse(s)
			#data = (s.recv(self.total_buffer))
			data.begin()									
			s.shutdown(1)
			s.close()
			print TEST_CASE_RESULT_TEMPLATE % ("RD Service HTTP " +request_verb + " disabled verification " , TEST_CASE_RESULT_FAILURE)
		except socket.error,exc:
			print TEST_CASE_RESULT_TEMPLATE % ("RD Service HTTP " +request_verb + " disabled verification ", TEST_CASE_RESULT_PASS)		                                                
예제 #35
0
파일: __init__.py 프로젝트: bitkapp/traidor
    def _do_handshake(self, url, version=6, origin=None, cookie=None):
        """http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-06"""
        urlParts = urlparse.urlparse(url)
        key = b64encode(urandom(16))
        request  = "GET %s HTTP/1.1\r\n" \
                   "Host: %s\r\n" \
                   "Upgrade: WebSocket\r\n" \
                   "Connection: Upgrade\r\n" \
                   "Sec-WebSocket-Key: %s\r\n" \
                   "Sec-WebSocket-Version: %d\r\n" \
                   % (urlParts.path, self.host, key, version)

        if origin is not None:
            request += "Sec-WebSocket-Origin: %s\r\n" % (origin,)

        if cookie is not None:
            request += cookie.output(header="Cookie:") + "\r\n"

        # Authenticate if required
        if self.username is not None:
            request += "Authorization: Basic %s\r\n" \
                    % b64encode('%s:%s' % (self.username, self.password or ''))

        # Finish request
        request += "\r\n"
        if _debug:
            print >> sys.stderr, '\x1B[D\x1B[31m%s\x1B[m' % (request,),
        self.socket.sendall(request)

        response = HTTPResponse(self.socket)
        response.begin()

        if _debug:
            print >> sys.stderr, '\x1B[D\x1B[34m%s' % ({9: 'HTTP/0.9', 10: 'HTTP/1.0', 11: 'HTTP/1.1'}[response.version],), response.status, response.reason
            print >> sys.stderr, '%s\x1B[m' % (response.msg,)

        if response.status != 101:
            self.socket.close()
            raise RuntimeError("WebSocket upgrade failed: %d %s" % (response.status, response.reason))

        expected = sha1(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()
        assert len(expected) == 20
        expected = b64encode(expected)

        if 'Sec-WebSocket-Accept' not in response.msg:
            raise RuntimeError, "Expected WebSocket header not present: Sec-WebSocket-Accept"

        if response.msg['Sec-WebSocket-Accept'].strip() != expected:
            raise RuntimeError, "Invalid WebSocket accept returned: %s %s %s" % (key, expected, response.msg['Sec-WebSocket-Accept'])
예제 #36
0
	def connect_rd_services(self,port):
		try:
			s = self.get_socket(self.host,port)
			request_string = self.get_request_templates()
			#print request_string 
			s.sendall(request_string)
			data = HTTPResponse(s)
			#data = (s.recv(self.total_buffer))
			data.begin()									
			s.shutdown(1)
			s.close()
			return data
		except socket.error,exc:
			print "Error Connecting to Port " + str(port)
			return 0
예제 #37
0
def get(addr, path, debug=False):
    client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    client.connect(addr)
    client.send("GET {} HTTP/1.0\r\n\r\n".format(path))
    resp_str = client.recv(65536)
    source = FakeSocket(resp_str)
    resp = HTTPResponse(source)
    resp.begin()
    if resp.status == 200:
        text = resp.read(len(resp_str))
        data = json.loads(text)
        if debug:
            log.debug(data)
        return data

    return {}
예제 #38
0
def get(addr, path, debug=False):
    client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    client.connect(addr)
    client.send("GET {} HTTP/1.0\r\n\r\n".format(path))
    resp_str = client.recv(65536)
    source = FakeSocket(resp_str)
    resp = HTTPResponse(source)
    resp.begin()
    if resp.status == 200:
        text = resp.read(len(resp_str))
        data = json.loads(text)
        if debug:
            log.debug(data)
        return data

    return {}
예제 #39
0
    def _parse_http(self, text):
        # if the response text starts with a 302, skip to the next non-302 header
        if re.match(r'^HTTP/.*?\s302 Found', text):
            m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
            if not m:
                raise Exception("Unrecognized response: %s" % text)
            else:
                text = m.group(1)

        # if the response text starts with a "200 Connection established" but continues with a 201,
        # skip the 200 header. This happens when using a proxy.
        #
        # e.g. HTTP/1.1 200 Connection established
        #       Via: 1.1 proxy
        #       Connection: Keep-Alive
        #       Proxy-Connection: Keep-Alive
        #
        #       HTTP/1.1 201 Created
        #       Server: GitHub.com
        #       ...
        #       Status: 201 Created
        #       ...
        if re.match(r'^HTTP/.*?\s200 Connection established', text):
            m = re.search(
                r'(HTTP/\d+\.\d+\s(?!200 Connection established).*$)', text,
                re.S)
            if not m:
                raise Exception("Unrecognized response: %s" % text)
            else:
                text = m.group(1)

        # remove Transfer-Encoding: chunked header, as it causes reading the response to fail
        # first do a quick check for it, so we can avoid doing the expensive negative-lookbehind
        # regex if we don't need it
        if "Transfer-Encoding: chunked" in text:
            # we do the negative-lookbehind to make sure we only strip the Transfer-Encoding
            # string in the header
            text = re.sub(r'(?<!\r\n\r\n).*?Transfer-Encoding: chunked\r\n',
                          '',
                          text,
                          count=1)

        logger.debug("CurlSession - getting socket from %s" % text)
        socket = self.FakeSocket(text)
        response = HTTPResponse(socket)
        response.begin()
        return response
class PassThroughProxyDownloader(DownloaderBase):
    def __init__(self, user_agent, proxy):
        self.user_agent = user_agent

        web_proxy_split = proxy.split(':')
        self.proxy_ip = web_proxy_split[0]
        self.proxy_port = int(web_proxy_split[1])

    def download(self, url, referrer=None, if_modified_since=None, if_none_match=None):
        request = [
            "GET %s HTTP/1.1" % url,
            "User-Agent: %s" % self.user_agent,
            "Accept-Encoding: gzip, deflate, compress",
            "Accept: */*"
            "Host: %s" % urlparse(url).hostname,
            "Connection: Close"  # this may not be necessary
        ]

        for header, value in CommonHeaders.iteritems():
            request.append("%s: %s" % (header, value))

        for header, value in DownloaderBase.construct_headers(referrer, if_modified_since, if_none_match).iteritems():
            request.append("%s: %s" % (header, value))

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((self.proxy_ip, self.proxy_port))
        except socket.error, ex:
            raise ProxyDownException(ex)

        s.send("\r\n".join(request) + '\r\n\r\n')

        r = HTTPResponse(s, strict=False, method='GET', buffering=True)
        r.begin()

        resp = urllib3Response.from_httplib(r, decode_content=False)

        s.close()
        r.close()

        response = requests.Response()

        # Fallback to None if there's no status_code, for whatever reason.
        response.status_code = getattr(resp, 'status', None)
        response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
        response.encoding = get_encoding_from_headers(response.headers)
        response._content = resp.data
        response.raw = resp
        response.reason = response.raw.reason
        response.url = url

        # don't care about cookies right now
        #extract_cookies_to_jar(response.cookies, req, resp)

        # don't worry about the requests' Request object right now until it is needed
        #response.request = req
        response.request = None

        return response
예제 #41
0
def import_incidents(request, source):
    '''
    Master incident import view.  Used for the initial population of the dispatch database.
    '''
    if source == 'twitter':
        get_twitter_incidents(request, TWITTER_USERNAME)

        return HTTPResponse(status=200)

    elif source == 'email':
        get_email_incidents(request, EMAIL_USERNAME)

        return HTTPResponse(status=200)

    else:
        raise ValueError(
            "Invalid dispatch source.  Please specify twitter, email or cad.")
예제 #42
0
 def get_json(self, response):
     res_bytes = response.getResponse()  # byte[]
     if not res_bytes:
         print "[!] Failed to get json for Response. Response has not bytes"
         return {}
     res_str = str(helpers.bytesToString(res_bytes))
     source = FakeSocket(res_str)
     res_obj = HTTPResponse(source)
     res_obj.begin()
     # print "Reading response object"
     res_text = res_obj.read()
     try:
         res_data = json.loads(res_text)
     except Exception as e:
         print "[!] Failed to convert to json: " + str(res_text)
         res_data = {}
     return res_data
예제 #43
0
    def getheaders(self):
        """Return all headers from the response

        This gives the ability to access all HTTP headers of a discovery response.

        :return: dict[str, str]
        """
        return HTTPResponse.getheaders(self)
예제 #44
0
def http_has_valid_content_length(all_data):
    """
	Gets HTTP data, returns:
			1. True if it has a header "content_length" and its value <= MAX_HTTP_CONTENT_LENGTH,
			2. False otherwise (includes the case were all_data doesn't contain this header)
	NOTE: USE THIS ONLY ON HTTP DATA!
	"""
    source = FakeSocket(all_data)
    response = HTTPResponse(source)
    response.begin()

    content_len_value = int(response.getheader('content-length', -1))
    if content_len_value == -1 or content_len_value > MAX_HTTP_CONTENT_LENGTH:
        print("Invalid (or missing) content-length header.")
        return False

    return True
예제 #45
0
    def getheaders(self):
        """Return all headers from the response

        This gives the ability to access all HTTP headers of a discovery response.

        :return: dict[str, str]
        """
        return HTTPResponse.getheaders(self)
예제 #46
0
파일: rpc.py 프로젝트: hqlian007/QT4i
 def _parse_ws_response(self):
     http_response = self.__ws.recv()
     response = HTTPResponse(FakeSocket(http_response))
     response.begin()
     while response.length > len(http_response):
         http_response += self.__ws.recv()
     response = HTTPResponse(FakeSocket(http_response))
     self.__transport.verbose = 0
     response.begin()
     return self.__transport.parse_response(response)
    def _get_response(self):

        # Parse response
        h = HTTPResponse(self._proxy_sock)
        h.begin()
        res_d = {
            'status':
            h.status,
            'reason':
            h.reason,
            'req_version':
            self.request_version,
            'msg':
            h.msg,
            'trans_enc': (h.msg['Transfer-Encoding']
                          if 'Transfer-Encoding' in h.msg else '[NO-ENCODING]')
        }
        # Get rid of the pesky header
        del h.msg['Transfer-Encoding']

        res = '%(req_version)s %(status)s %(reason)s\r\n%(msg)s\r\n' % res_d
        res_d['data'] = h.read()
        res_d['time'] = time()
        res += res_d['data']
        # Let's close off the remote end
        h.close()
        self._proxy_sock.close()
        # Relay the message
        self.request.sendall(self.mitm_response(res))
        print res
        return res_d
예제 #48
0
 def __init__(self, payload):
     r = HTTPResponse(self._fakeSocket(payload))
     r.begin()
     self.st = r.getheader("st") or None
     self.usn = r.getheader("usn") or None
     self.server = r.getheader("server") or None
     self.location = r.getheader("location") or None
예제 #49
0
파일: dpi_sock.py 프로젝트: GNQG/lr2irproxy
    def send_and_recv(self):
        try:
            # because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
            # send message to www.dream-pro.info with socket.socket to make
            # http connection
            sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            sock.connect((WWW_DREAM_PRO_INFO,80))
            sock.sendall(str(self))
        except:
            print 'SocketError'
            return

        res = HTTPResponse(sock)
        res.begin()
        res_body = res.read()
        res.close()

        if 'transfer-encoding' in res.msg:
            # httplib.HTTPResponse automatically concatenate chunked response
            # but do not delete 'transfer-encoding' header
            # so the header must be deleted
            res.msg.__delitem__('transfer-encoding')
        compmeth = res.msg.getheader('content-encoding','').lower()
        if compmeth and compmeth.find('identity') != 0 :
            # response body is compressed with some method
            offset = 0
            if compmeth.find('gzip') != -1:
                # if body is gziped, header offset value is 47
                # if not, offset value is 0
                # this server does not support sdch...
                offset += 47
            res_body = decompress(res_body,offset)
            res.msg['content-encoding'] = 'identity'

        return res, res_body
예제 #50
0
def parse_response(response_text):
    """
        Given an HTTP response line and headers, return a requests.Response object.
    """
    class FakeSocket():
        def __init__(self, response_str):
            self._file = StringIO(response_str)

        def makefile(self, *args, **kwargs):
            return self._file

    source = FakeSocket(response_text)
    response = HTTPResponse(source)
    response.begin()
    requests_response = requests.Response()
    requests_response.status_code = response.status
    requests_response.headers = CaseInsensitiveDict(response.getheaders())
    return requests_response
def SOCKET_SEND(http_packet):
		# SEND REQUEST
		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		ssl_s = ssl.wrap_socket(s)
		ssl_s.connect((host_address, https_port))
		if SHOW_HTTP_REQUESTS: print ('--- Sending ---\r\n' + http_packet + '\r\n----')
		ssl_s.send(http_packet)

		# GET RESPONSE
		response = ssl_s.recv(1024)
		ssl_s.close()
		if SHOW_HTTP_REQUESTS: print ('--- Response --- \r\n' + str(response) + '\r\n---')

		#PARSE REPONSE
		fake_socket_response = FakeSocket(response)
		parsed_response = HTTPResponse(fake_socket_response)
		parsed_response.begin()
		return parsed_response
예제 #52
0
파일: htstream.py 프로젝트: erh/simples3
 def _read_resp(self):
     # Let httplib handle the response part.
     from httplib import HTTPResponse
     from urllib import addinfourl
     resp = HTTPResponse(self.sock, strict=True, method=self.method)
     resp.begin()
     if resp.will_close:
         self.sock.close()
     else:
         # TODO Should make something of this, I suppose.
         self._active_resp = resp
     # Courtesy of urllib2, weird stuff going on here.
     resp.recv = resp.read
     fp = socket._fileobject(resp, close=True)
     rv = addinfourl(fp, resp.msg, self.get_full_url())
     rv.code = resp.status
     rv.msg = resp.reason
     return rv
예제 #53
0
    def testWithoutCRLF(self):
        # Tests the use of just newlines rather than CR/LFs.
        data = "Echo\nthis\r\nplease"
        s = ("GET / HTTP/1.0\n"
             "Connection: close\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)

        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        length = int(response.getheader('Content-Length', '0'))
        response_body = response.read(length)
        self.failUnlessEqual(length, len(data))
        self.failUnlessEqual(response_body, data)
예제 #54
0
 def _parse_http(self, text):
     # if the response text starts with a 302, skip to the next non-302 header
     if re.match(r'^HTTP/.*?\s302 Found', text):
         m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
         if not m:
             raise Exception("Unrecognized response: %s" % text)
         else:
             text = m.group(1)
     # remove Transfer-Encoding: chunked header, as it causes reading the response to fail
     # first do a quick check for it, so we can avoid doing the expensive negative-lookbehind
     # regex if we don't need it
     if "Transfer-Encoding: chunked" in text:
         # we do the negative-lookbehind to make sure we only strip the Transfer-Encoding
         # string in the header
         text = re.sub(r'(?<!\r\n\r\n).*?Transfer-Encoding: chunked\r\n', '', text, count=1)
     socket = self.FakeSocket(text)
     response = HTTPResponse(socket)
     response.begin()
     return response
예제 #55
0
    def _parse_http(self, text):
        # if the response text starts with a 302, skip to the next non-302 header
        if re.match(r'^HTTP/.*?\s302 Found', text):
            m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
            if not m:
                raise Exception("Unrecognized response: %s" % text)
            else:
                text = m.group(1)

        # if the response text starts with a "200 Connection established" but continues with a 201,
        # skip the 200 header. This happens when using a proxy.
        #
        # e.g. HTTP/1.1 200 Connection established
        #       Via: 1.1 proxy
        #       Connection: Keep-Alive
        #       Proxy-Connection: Keep-Alive
        #
        #       HTTP/1.1 201 Created
        #       Server: GitHub.com
        #       ...
        #       Status: 201 Created
        #       ...
        if re.match(r'^HTTP/.*?\s200 Connection established', text):
            m = re.search(r'(HTTP/\d+\.\d+\s(?!200 Connection established).*$)', text, re.S)
            if not m:
                raise Exception("Unrecognized response: %s" % text)
            else:
                text = m.group(1)

        # remove Transfer-Encoding: chunked header, as it causes reading the response to fail
        # first do a quick check for it, so we can avoid doing the expensive negative-lookbehind
        # regex if we don't need it
        if "Transfer-Encoding: chunked" in text:
            # we do the negative-lookbehind to make sure we only strip the Transfer-Encoding
            # string in the header
            text = re.sub(r'(?<!\r\n\r\n).*?Transfer-Encoding: chunked\r\n', '', text, count=1)

        logger.debug("CurlSession - getting socket from %s" % text)
        socket = self.FakeSocket(text)
        response = HTTPResponse(socket)
        response.begin()
        return response
예제 #56
0
파일: helper.py 프로젝트: zeulb/Webproxy
  def __init__(self, response_str):
    source = FakeSocket(response_str)
    HTTPResponse.__init__(self, source)
    self.begin()



# Request
# error_code 
# command    GET/POST
# path       PATH
# request_version "HTTP/1.0"
# headers   "Headers"
# headers.keys() 
# headers['host']

# Response
# status    CODE
# getheader('Content-Length')
# read(len(response_str)) 
예제 #57
0
def parse_http_response(sock):

    try:
        # H4ck to standardize the API between sockets and SSLConnection objects
        response = sock.read(4096)
    except AttributeError:
        response = sock.recv(4096)

    if 'HTTP/' not in response:
        # Try to get the rest of the response
        try:
            response += sock.read(4096)
        except AttributeError:
            response += sock.recv(4096)

    fake_sock = FakeSocket(response)
    response = HTTPResponse(fake_sock)
    response.begin()

    return response
예제 #58
0
def parse_response(response_text):
    """
        Given an HTTP response line and headers, return a requests.Response object.
    """
    class FakeSocket():
        def __init__(self, response_str):
            self._file = StringIO(response_str)

        def makefile(self, *args, **kwargs):
            return self._file

    source = FakeSocket(response_text)
    response = HTTPResponse(source)
    response.begin()
    requests_response = requests.Response()
    requests_response.status_code = response.status

    headers = CaseInsensitiveDict(response.getheaders())
    # Reset headers['x-robots-tag'], so that we can handle the
    # possibilility that multiple x-robots directives might be included
    # https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
    # e.g.
    # HTTP/1.1 200 OK
    # Date: Tue, 25 May 2010 21:42:43 GMT
    # (...)
    # X-Robots-Tag: googlebot: nofollow
    # X-Robots-Tag: otherbot: noindex, nofollow
    # (...)
    # Join with a semi-colon, not a comma, so that multiple agents can
    # be recovered. As of 12/14/16, there doesn't appear to be any spec
    # describing how to do this properly (since commas don't work).
    # Since parsed response headers aren't archived, this convenience is
    # fine. However, it's worth keeping track of the situation.
    robots_directives = []
    for directive in response.msg.getallmatchingheaders('x-robots-tag'):
        robots_directives.append(directive.split(": ", 1)[1].replace("\n", "").replace("\r", ""))
    headers['x-robots-tag'] = ";".join(robots_directives)

    requests_response.headers = headers

    return requests_response
예제 #59
0
    def process_record(self, record):
        content = None
        try:
            payload = record.payload.read()
            s = FakeSocket(payload)
            response = HTTPResponse(s)
            response.begin()
            status_code = response.status
            if status_code != 200:
                return
            content_type = response.getheader('Content-Type', '')
            if 'text/html' not in content_type:
                return

            headers = response.getheaders()
            content = response.read(len(payload))
        except Exception:
            self.increment_counter('errors', 'process_record', 1)
            logging.error('Error processing record: {}', traceback.format_exc())
            return

        if content is not None:
            content = content.strip()

        if not content:
            return

        for item in self.process_content(record.url, headers, content):
            yield item