def read_get(self): reader = SocketReader(self.sock) resp = HttpStream(reader) status_code = resp.status_code() headers = dict(resp.headers()) body = resp.body_string() return {"status_code": status_code, "headers": headers, "body": body}
def create(server, client, config): client.setblocking(1) request = HttpStream(SocketReader(client)) body_stream = request.body_file() environ = request.wsgi_environ() ''' the environ produced by Python and C parser is not consistent we have to set some missing values here ''' host, port = server.getsockname() environ.update({ 'SERVER_NAME': host, 'SERVER_PORT': str(port), 'wsgi.version': (1, 0), 'wsgi.url_scheme': utils.guess_scheme(environ), 'wsgi.input': body_stream, 'wsgi.multithread': False, 'wsgi.multiprocess': config['workers'] > 1, 'wsgi.run_once': False, 'wsgi.file_wrapper': utils.FileWrapper, }) response = Response(request, client) return environ, response
def test_parse_from_real_socket(): # would fail on python2.6 before the recv_into hack sock, sink = socket.socketpair() sink.send(complete_request) reader = SocketReader(sock) stream = HttpStream(reader) assert stream.headers()
def http_serve(config, stdin, stdout): # Re-open the stdin and stdout file descriptors using raw unbuffered i/o. stdin = io.FileIO(stdin.fileno()) stdout = io.FileIO(stdout.fileno(), 'w') # Parse an HTTP request on stdin. parser = HttpStream(stdin, kind=HTTP_REQUEST) wsgi_env = parser.wsgi_environ() request_method = wsgi_env['REQUEST_METHOD'] request_path = wsgi_env['PATH_INFO'] request_length = wsgi_env.get('HTTP_CONTENT_LENGTH', 0) request_body = parser.body_file() try: success = False if request_method != 'GET': raise HttpError(405) if request_path.endswith('/'): http_respond(stdout, 301, location=request_path.rstrip('/'), body='') else: path = resolve_path(request_path) mimetype = guess_type(path)[0] try: output = render(config, request_path) http_respond(stdout, 200, mimetype=mimetype, body=output) except TemplateNotFound as e: # Retry as static file. full_static_path = config.static_root + path with file(full_static_path) as static_file: http_respond(stdout, 200, mimetype=mimetype, body=static_file) success = True except InvalidRequestPath as e: http_respond(stdout, 400) except HttpError as e: http_respond(stdout, e.status) except IOError as e: if e.errno == errno.ENOENT: http_respond(stdout, 404) elif e.errno == errno.EACCES: http_respond(stdout, 403) else: http_respond(stdout, 500) except: http_respond(stdout, 500) return success
def send_downstream(self, p, rhost, rport, method, url, upsock): with self.connect_downstream(rhost, rport) as downsock: request = '{method} {url} HTTP/{version[0]}.{version[1]}\r\n{headers}\r\n\r\n'.format(method=method.decode(), url=url, version=p.version(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in p.headers().items()])) self.logger.debug('sending request {0!r}'.format(request)) downsock.send(request.encode()) downstream = HttpStream(SocketReader(downsock)) self.logger.debug('response: header={0}'.format(downstream.headers())) upsock.send('HTTP/{version[0]}.{version[1]} {code} {status}\r\n{headers}\r\n\r\n'.format(version=downstream.version(), code=downstream.status_code(), status=downstream.status(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in downstream.headers().items()])).encode()) upsock.send(downstream.body_string())
def handle(self, sock, addr): p = HttpStream(SocketReader(sock)) path = p.path() data = "hello world" sock.send("".join([ "HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n", "Content-Length:" + str(len(data)) + "\r\n", "Connection: close\r\n\r\n", data ]))
def main(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect(('gunicorn.org', 80)) s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n") p = HttpStream(SocketReader(s)) print(p.headers()) print(p.body_file().read()) finally: s.close()
def handle(self, sock, addr): p = HttpStream(SocketReader(sock)) path = p.path() data = "welcome wold" sock.send("".join(["HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n", "Content-Length:" + str(len(data)) + "\r\n", "Connection: close\r\n\r\n", data]))
def main(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect(('gunicorn.org', 80)) s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n") p = HttpStream(SocketReader(s)) print p.headers() print p.body_file().read() finally: s.close()
def recv(self): try: stream = HttpStream(self.reader, kind=HTTP_RESPONSE, parser_class=HttpParser, decompress=True) status = stream.status_code() version = stream.version() fields = stream.headers() content = stream.body_file() self.version = 'HTTP/%s.%s' % version return status, fields, content except NoMoreData: pass
def recPOSTheader(HOST, request_headerPOST): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((HOST, PORT)) try: s.sendall(request_headerPOST.encode('utf-8')) except: print("An error has occured while trying to send the POST request") dataReceived = SocketReader(s) parsed = HttpStream(dataReceived) header = parsed.headers() return header
def main(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect(('baike.baidu.com', 80)) s.send(b("GET /view/262.htm HTTP/1.1\r\nHost: baike.baidu.com\r\n\r\n")) p = HttpStream(SocketReader(s), decompress=True) print(p.headers()) print(p.body_file().read()) finally: s.close()
def proxy_connection(self, request, req_addr, is_ssl): """ do the proxy connection """ proxy_settings = os.environ.get('%s_proxy' % request.parsed_url.scheme) if proxy_settings and proxy_settings is not None: request.is_proxied = True proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) addr = parse_netloc(urlparse.urlparse(proxy_settings)) if is_ssl: if proxy_auth: proxy_auth = 'Proxy-authorization: %s' % proxy_auth proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr user_agent = request.headers.iget('user_agent') if not user_agent: user_agent = "User-Agent: restkit/%s\r\n" % __version__ proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) conn = self._pool.get(host=addr[0], port=addr[1], pool=self._pool, is_ssl=is_ssl, extra_headers=[], **self.ssl_args) conn.send(proxy_pieces) p = HttpStream(SocketReader(conn.socket()), kind=1, decompress=True) if p.status_code != 200: raise ProxyError("Tunnel connection failed: %d %s" % (resp.status_int, body)) _ = p.body_string() else: headers = [] if proxy_auth: headers = [('Proxy-authorization', proxy_auth)] conn = self._pool.get(host=addr[0], port=addr[1], pool=self._pool, is_ssl=False, extra_headers=[], **self.ssl_args) return conn return
def recGETbody(HOST, request_headerGET): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((HOST, PORT)) try: s.sendall(request_headerGET.encode('utf-8')) except: print("An error has occured while trying to send the GET request") dataReceived = SocketReader(s) parsed = HttpStream(dataReceived) body = parsed.body_file().read().decode('utf-8') return body
def main(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect(('baike.baidu.com', 80)) s.send( b("GET /view/262.htm HTTP/1.1\r\nHost: baike.baidu.com\r\n\r\n")) p = HttpStream(SocketReader(s), decompress=True) print(p.headers()) print(p.body_file().read()) finally: s.close()
def GETverbose(HOST, request_headerGET): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((HOST, 80)) try: s.sendall(request_headerGET.encode('utf-8')) except: print("An error has occured while trying to send the GET request") dataReceived = SocketReader(s) parsed = HttpStream(dataReceived) body = parsed.body_file().read().decode('utf-8') header = parsed.headers() return pprint.pformat(header) + body
def rewrite_response(resp): try: while True: parser = HttpStream(resp) # we aren't doing anything here for data in parser: resp.writeall(data) if not parser.should_keep_alive(): # close the connection. break except (socket.error, NoMoreData): pass
def handle(self, upsock, peer): with closing(upsock) as upsock: origaddr = upsock.getsockname() self.logger.debug('intercepted connection from {0} to {1}'.format(peer, origaddr)) p = HttpStream(SocketReader(upsock)) self.logger.debug('request: method={0} url={1} version={2} headers={3}'.format(p.method(), p.url(), p.version(), p.headers())) url = None if p.method() == b'CONNECT': self.handle_connect(p, upsock) elif p.url()[0] == '/': self.handle_transparent(p, upsock) else: self.handle_proxy(p, upsock)
def proxy_connection(self, request, req_addr, is_ssl): """ do the proxy connection """ proxy_settings = os.environ.get('%s_proxy' % request.parsed_url.scheme) if proxy_settings and proxy_settings is not None: request.is_proxied = True proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) addr = parse_netloc(urlparse.urlparse(proxy_settings)) if is_ssl: if proxy_auth: proxy_auth = 'Proxy-authorization: %s' % proxy_auth proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr user_agent = request.headers.iget('user_agent') if not user_agent: user_agent = "User-Agent: restkit/%s\r\n" % __version__ proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) conn = self._session.get(host=addr[0], port=addr[1], pool=self._session, is_ssl=is_ssl, extra_headers=[], **self.ssl_args) conn.send(proxy_pieces) p = HttpStream(SocketReader(conn.socket()), kind=1, decompress=True) if p.status_code != 200: raise ProxyError("Tunnel connection failed: %d %s" % (resp.status_int, body)) _ = p.body_string() else: headers = [] if proxy_auth: headers = [('Proxy-authorization', proxy_auth)] conn = self._session.get(host=addr[0], port=addr[1], pool=self._session, is_ssl=False, extra_headers=[], **self.ssl_args) return conn return
def _handle_client(self, client, address): try: r = SocketReader(client) p = HttpStream(r) headers = p.headers() body = "" if "CONTENT-LENGTH" in headers and int( headers["CONTENT-LENGTH"]) > 0: body = p.body_string(binary=False) # System requests if p.path() == "/alive": self._respond(client, "ok") return # User requests req = IpcServer.Request(self._respond, client, p.path(), p.method(), headers, body) if p.path() == "/message": req.respond() self.requests.put(req) except Exception as e: client.close() raise e
def recv(self, wsgi=False): try: stream = HttpStream(self.reader, kind=HTTP_REQUEST, parser_class=HttpParser, decompress=True) if bool(wsgi): environ = stream.wsgi_environ() environ['wsgi.url_scheme'] = guess_scheme(environ) environ['wsgi.input'] = stream.body_file() environ['wsgi.socket'] = self.socket return environ # BUG: # http-parser has an issue here, if we call 'method' before 'headers' # and invalid method name is returned... fields = stream.headers() method = stream.method() url = stream.url() version = stream.version() content = stream.body_file() url = urlparse(url) path = url.path query = parse_qs(url.query, keep_blank_values=True) fragment = url.fragment for k, v in six.iteritems(dict(query)): query[k] = parse_query_value(v) self.version = 'HTTP/%s.%s' % version return method, path, query, fragment, fields, content except NoMoreData: pass
def handle(self): self.logger = logging.getLogger("proxy") self.session = Session() try: request = HttpStream(SocketReader(self.request)) _ = request.status() except http_parser.http.BadStatusLine: return self.logger.warning(request.path()) if 'CONNECT' == request.method(): self.https_proxy(request) else: self.http_proxy(request) self.session.close()
def rewrite_request(req): try: while True: parser = HttpStream(req) new_headers = rewrite_headers(parser, dict({'Host': 'api.mailgun.net'}, **basic_auth_headers('api', MAILGUN_APIKEY))) if new_headers is None: break req.send(new_headers) body = parser.body_file() while True: data = body.read(8192) if not data: break req.writeall(data) except (socket.error, NoMoreData): pass
def rewrite_request(req): try: while True: parser = HttpStream(req) new_headers = rewrite_headers(parser, {'Host': 'gunicorn.org'}) if new_headers is None: break req.send(new_headers) body = parser.body_file() while True: data = body.read(8192) if not data: break req.writeall(data) except (socket.error, NoMoreData): pass
def processData(newsocketconn, fromaddr, context, ipaddr, q): if useSSL: connstreamout = context.wrap_socket(newsocketconn, server_side=True) else: connstreamout = newsocketconn try: try: ### Read the json response using Socket Reader and split header and body r = SocketReader(connstreamout) p = HttpStream(r) headers = p.headers() q.put(makeLogMessage("INFO", "idrac", "ProcessData")) if p.method() == 'POST': bodydata = p.body_file().read() bodydata = bodydata.decode("utf-8", errors='ignore') ### Read the json response and print the output # outdata = dict() try: with open(f"{ipaddr[3]}/output/{ipaddr[2]}", "a+") as fd: fd.write(bodydata) fd.write("\n") except Exception as ex: print("Exception Occured =", ex) StatusCode = """HTTP/1.1 200 OK\r\n\r\n""" connstreamout.send(bytes(StatusCode, 'UTF-8')) # if p.method() == 'GET': # res = "HTTP/1.1 200 OK\n" \ # "Content-Type: application/json\n" \ # "\n" + json.dumps(data_buffer) # connstreamout.send(res.encode()) # data_buffer.clear() except Exception as err: outdata = connstreamout.read() traceback.print_exc() print("Data needs to read in normal Text format.") finally: connstreamout.shutdown(socket.SHUT_RDWR) connstreamout.close()
def get_response(self, request, connection): """ return final respons, it is only accessible via peform method """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to parse response") p = HttpStream(SocketReader(connection.socket()), kind=1, decompress=self.decompress) if log.isEnabledFor(logging.DEBUG): log.debug("Got response: %s %s" % (p.version(), p.status())) log.debug("headers: [%s]" % p.headers()) location = p.headers().get('location') if self.follow_redirect: if p.status_code() in ( 301, 302, 307, ): connection.close() if request.method in ('GET', 'HEAD',) or \ self.force_follow_redirect: if hasattr(self.body, 'read'): try: self.body.seek(0) except AttributeError: raise RequestError( "Can't redirect %s to %s " "because body has already been read" % (self.url, location)) return self.redirect(location, request) elif p.status_code() == 303 and self.method == "POST": connection.close() request.method = "GET" request.body = None return self.redirect(location, request) # create response object resp = self.response_class(connection, request, p) # apply response filters for f in self.response_filters: f.on_response(resp, request) if log.isEnabledFor(logging.DEBUG): log.debug("return response class") # return final response return resp
def rewrite_response(self, resp, extra): try: if extra.get('rewrite_response', False): parser = HttpStream(resp, decompress=True) rw = RewriteResponse(parser, resp, extra) rw.execute() else: parser = HttpStream(resp) headers = parser.headers() headers['connection'] = 'close' new_headers = headers_lines(parser, headers) resp.writeall("".join(new_headers) + "\r\n") body = parser.body_file() send_body(resp, body, parser.is_chunked()) except (socket.error, NoMoreData, ParserError): pass
def handle(self): h = SocketReader(self.request) p = HttpStream(h) #delagate=url[p.url()] delagate = Server.urls[p.url()] data = p.body_string() if data: data = data.decode('utf-8') else: data = "" js = None data = self.createData(data) print(p.url(), data) try: res = delagate(self.request, data) #todo change to: # self.request.send(res) except Exception as ex: print(ex)
def rewrite_request(req): log = logging.getLogger("rewrite_request") peername = req._src.getpeername() try: while True: request_id = str(uuid.uuid4()) parser = HttpStream(req) new_headers = rewrite_headers( parser, peername, {'x-nimbus-io-user-request-id': request_id}) if new_headers is None: break log.debug("rewriting request %s" % (request_id, )) req.send(new_headers) body = parser.body_file() while True: data = body.read(8192) if not data: break req.writeall(data) except (socket.error, NoMoreData): pass
def rewrite_request(req): log = logging.getLogger("rewrite_request") peername = req._src.getpeername() try: while True: request_id = str(uuid.uuid4()) parser = HttpStream(req) new_headers = rewrite_headers( parser, peername, {'x-nimbus-io-user-request-id': request_id}) if new_headers is None: break log.debug("rewriting request %s" % ( request_id, )) req.send(new_headers) body = parser.body_file() while True: data = body.read(8192) if not data: break req.writeall(data) except (socket.error, NoMoreData): pass
def join_data(request: HttpStream, type_: str): assert type_ in ('Request', 'Response') # try: major, minor = request.version() # except ParserError: # raise if type_ == 'Request': url = request.url() start_line = f'{request.method()} {url} HTTP/{major}.{minor}' else: start_line = f'HTTP/{major}.{minor} {request.status()}' body = request.body_string(binary=True) headers = request.headers() setcookies = headers['Set-Cookie'] if 'Set-Cookie' in headers else None if "Content-Encoding" in headers: assert headers["Content-Encoding"] == "br" body = brotli.decompress(body) for key in ['Transfer-Encoding', 'Content-Encoding', 'Content-Length', 'Set-Cookie']: if key in headers: del headers[key] headers['Content-Length'] = len(body) headers = [f'{key}: {value}' for key, value in headers.items()] if setcookies is not None: setcookies = setcookies.split(",") ind = 0 while ind < len(setcookies): if setcookies[ind].lower()[:-4].endswith('expires'): headers.append(f"Set-Cookie: {setcookies[ind]}, {setcookies[ind + 1]}") ind += 2 else: headers.append(f"Set-Cookie: {setcookies[ind]}") ind += 1 headers = END_ONE_HEADER.join(headers) return (start_line + END_ONE_HEADER + headers + END_HEADERS).encode('ISO-8859-1') + body
def rewrite_request(self, req, extra): try: while True: if extra.get('rewrite_location', True): parser = HttpStream(req) prefix = extra.get('prefix', '') location = parser.url() if prefix: try: location = location.split(prefix, 1)[1] or '/' except IndexError: pass headers = rewrite_headers(parser, location, [('host', extra.get('host'))]) if headers is None: break extra['path'] = parser.path() req.writeall(headers) body = parser.body_file() while True: data = body.read(8192) if not data: break req.writeall(data) else: while True: data = req.read(io.DEFAULT_BUFFER_SIZE) if not data: break req.writeall(data) except (socket.error, NoMoreData): pass
def repeat_request(req_id): req = session.query(Request).filter(Request.id == req_id).limit(1).all() if len(req) == 0: return redirect(url_for('requests', error_message=f'Not found ID {req_id}')) req = req[0] sock_server = socket.create_connection((req.host, 443)) sock_server = ssl.create_default_context().wrap_socket(sock_server, server_hostname=req.host) server_reader = SocketReader(sock_server) sock_server.sendall(req.headers) server_resp = HttpStream(server_reader, decompress=True) req = req.headers.decode().split('\r\n') answer = join_data(server_resp, 'Response') answer = answer.decode(errors='ignore') answer = answer.split('\r\n') return render_template('repeat.html', req_id=req_id, req=req, answer=answer)
def get_response(self, request, connection): """ return final respons, it is only accessible via peform method """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to parse response") p = HttpStream(SocketReader(connection.socket()), kind=1, decompress=self.decompress) if log.isEnabledFor(logging.DEBUG): log.debug("Got response: %s %s" % (p.version(), p.status())) log.debug("headers: [%s]" % p.headers()) location = p.headers().get('location') if self.follow_redirect: if p.status_code() in (301, 302, 307,): connection.close() if request.method in ('GET', 'HEAD',) or \ self.force_follow_redirect: if hasattr(self.body, 'read'): try: self.body.seek(0) except AttributeError: raise RequestError("Can't redirect %s to %s " "because body has already been read" % (self.url, location)) return self.redirect(location, request) elif p.status_code() == 303 and self.method == "POST": connection.close() request.method = "GET" request.body = None return self.redirect(location, request) # create response object resp = self.response_class(connection, request, p) # apply response filters for f in self.response_filters: f.on_response(resp, request) if log.isEnabledFor(logging.DEBUG): log.debug("return response class") # return final response return resp
def __init__(self, host, ip, port, path, response_code, response, original_header): self.host = host self.ip = ip self.port = port self.path = path self.response_code = response_code self.response = response self.decompressed_response = self.decompress(response) self.parsed_response = HttpStream(StringReader(self.decompressed_response)) self.original_header = original_header self.arc1_record = None try: self.arc1_record = self.create_arc1_record() except NoMoreData: logging.warning(" HTTP parser failed on empty header '%s'", self.decompressed_response) except ParserError: logging.warning(" HTTP parser failed on '%s'", self.decompressed_response) except KeyError, k: logging.warning(" Couldn't find key %s in %s", k, self.parsed_response.headers().keys()) logging.warning(" " + self.decompressed_response)
def rewrite_request(req): try: while True: parser = HttpStream(req) headers = parser.headers() parsed_url = urlparse.urlparse(parser.url()) is_ssl = parsed_url.scheme == "https" host = get_host(parse_address(parsed_url.netloc, 80), is_ssl=is_ssl) headers['Host'] = host headers['Connection'] = 'close' if 'Proxy-Connection' in headers: del headers['Proxy-Connection'] location = urlparse.urlunparse( ('', '', parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) httpver = "HTTP/%s" % ".".join(map(str, parser.version())) new_headers = [ "%s %s %s\r\n" % (parser.method(), location, httpver) ] new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \ for hname, hvalue in headers.items()]) req.writeall(bytes("".join(new_headers) + "\r\n")) body = parser.body_file() send_body(req, body, parser.is_chunked()) except (socket.error, NoMoreData, ParserError): pass
def https_proxy(self, connectreq: HttpStream): self.request.sendall(CON_ESTABLISH) host, port = connectreq.path().split(':') cert_path = self.generate_cert(host) # request to localhost as server try: sock_client = ssl.wrap_socket(self.request, keyfile="cert.key", certfile=cert_path, ssl_version=ssl.PROTOCOL_TLS, server_side=True) sock_server = socket.create_connection((host, port)) sock_server = ssl.create_default_context().wrap_socket(sock_server, server_hostname=host) except OSError: return client_reader = SocketReader(sock_client) server_reader = SocketReader(sock_server) while True: # request to mail.ru as client clientreq = HttpStream(client_reader, decompress=True) try: clientreq_tosend = join_data(clientreq, 'Request') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return sock_server.sendall(clientreq_tosend) # request from mail.ru to localhost as client server_resp = HttpStream(server_reader, decompress=True) try: server_resp_tosend = join_data(server_resp, 'Response') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return try: sock_client.sendall(server_resp_tosend) except BrokenPipeError: return req = Request(clientreq.method(), server_resp.status_code(), clientreq_tosend, host) self.add_date_base(clientreq, req)
def send(self, protocol ,host, header): h = Http() h.build_header() data = header str_response_body="" str_response_header="" if protocol == "http": s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((host,80)) s.send(data.encode('utf-8')) """ response = b"" while True: d = client.recv(1024) response = response + d if not d: break #print(response) """ p = HttpStream(SocketReader(s)) elif protocol == "https": s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((host, 443)) s = ssl.wrap_socket(s, keyfile=None, certfile=None, server_side=False, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_SSLv23) s.sendall(data.encode('utf-8')) """ response = b"" while True: d = s.recv(4096) response = response + d if not d: break """ p = HttpStream(SocketReader(s)) #return html.escape(str(response, errors='replace')) # Transfer-Encoding: chunked の対処をする。 str_response_header += p.status()+"\n" for h in p.headers(): str_response_header += h+": "+p.headers()[h]+"\n" str_response_body += str(p.body_file().read(), errors='replace') return {"body":str_response_body, "header":str_response_header}
def rewrite_request(req): try: while True: parser = HttpStream(req) headers = parser.headers() parsed_url = urlparse.urlparse(parser.url()) is_ssl = parsed_url.scheme == "https" host = get_host(parse_address(parsed_url.netloc, 80), is_ssl=is_ssl) headers['Host'] = host headers['Connection'] = 'close' if 'Proxy-Connection' in headers: del headers['Proxy-Connection'] location = urlparse.urlunparse(('', '', parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) httpver = "HTTP/%s" % ".".join(map(str, parser.version())) new_headers = ["%s %s %s\r\n" % (parser.method(), location, httpver)] new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \ for hname, hvalue in headers.items()]) req.writeall(bytes("".join(new_headers) + "\r\n")) body = parser.body_file() send_body(req, body, parser.is_chunked()) except (socket.error, NoMoreData, ParserError): pass
def http_proxy(self, connectreq: HttpStream): reqheaders = connectreq.headers() # if ":" in connectreq.path(): parsed_url = connectreq.url().split('//')[1].split(':') if len(parsed_url) > 1: host = parsed_url[0] port = parsed_url[1].split('/')[0] else: host, port = reqheaders['host'], 80 if 'Proxy-Connection' in reqheaders: del reqheaders['Proxy-Connection'] elif 'proxy-connection' in reqheaders: del reqheaders['proxy-connection'] # request to mail.ru as client sock_server = socket.create_connection((host, port)) try: clientreq = join_data(connectreq, 'Request') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return sock_server.sendall(clientreq) # request from mail.ru to localhost as client if connectreq.method() == 'HEAD': return server_reader = SocketReader(sock_server) server_resp = HttpStream(server_reader, decompress=True) try: server_resp_tosend = join_data(server_resp, 'Response') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return self.request.sendall(server_resp_tosend) req = Request(connectreq.method(), server_resp.status_code(), clientreq, host) self.add_date_base(connectreq, req)
def test_upgrade_header(): stream = io.BytesIO(b'GET /test HTTP/1.1\r\nConnection: keep-alive, upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: hojIvDoHedBucveephosh8==\r\nSec-WebSocket-Version: 13\r\n\r\n') hdr = HttpStream(stream) hdr.headers() assert hdr.parser.is_upgrade() == 1 stream = io.BytesIO(b'GET /test HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: hojIvDoHedBucveephosh8==\r\nSec-WebSocket-Version: 13\r\n\r\n') hdr = HttpStream(stream) hdr.headers() assert hdr.parser.is_upgrade() == 1 stream = io.BytesIO(b'GET /test HTTP/1.1\r\nConnection: shenanigans\r\n\r\n') hdr = HttpStream(stream) hdr.headers() assert hdr.parser.is_upgrade() == 0
def test_repeated_header(): stream = io.BytesIO(b'GET /test HTTP/1.1\r\nX-Test: foo\r\nX-Test: bar\r\n\r\n') hdr = HttpStream(stream).headers() assert hdr['X-Test'] == 'foo, bar'
def test_repeated_continuation_header_py(): stream = io.BytesIO(b'GET /test HTTP/1.1\r\nX-Test: foo\r\n bar\r\nX-Test: baz\r\n qux\r\n\r\n') hdr = HttpStream(stream, parser_class=PyHttpParser).headers() assert hdr['X-Test'] == 'foo bar, baz qux'
def perform(self, request): """ perform the request. If an error happen it will first try to restart it """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to perform request: %s %s %s" % (request.host, request.method, request.path)) tries = 0 while True: conn = None try: # get or create a connection to the remote host conn = self.get_connection(request) # send headers msg = self.make_headers_string(request, conn.extra_headers) # send body if request.body is not None: chunked = request.is_chunked() if request.headers.iget('content-length') is None and \ not chunked: raise RequestError( "Can't determine content length and " + "Transfer-Encoding header is not chunked") # handle 100-Continue status # http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3 hdr_expect = request.headers.iget("expect") if hdr_expect is not None and \ hdr_expect.lower() == "100-continue": conn.send(msg) msg = None p = HttpStream(SocketReader(conn.socket()), kind=1, decompress=True) if p.status_code != 100: self.reset_request() if log.isEnabledFor(logging.DEBUG): log.debug("return response class") return self.response_class(conn, request, p) chunked = request.is_chunked() if log.isEnabledFor(logging.DEBUG): log.debug("send body (chunked: %s)" % chunked) if isinstance(request.body, types.StringTypes): if msg is not None: conn.send(msg + to_bytestring(request.body), chunked) else: conn.send(to_bytestring(request.body), chunked) else: if msg is not None: conn.send(msg) if hasattr(request.body, 'read'): if hasattr(request.body, 'seek'): request.body.seek(0) conn.sendfile(request.body, chunked) else: conn.sendlines(request.body, chunked) if chunked: conn.send_chunk("") else: conn.send(msg) return self.get_response(request, conn) except socket.gaierror, e: if conn is not None: conn.release(True) raise RequestError(str(e)) except socket.timeout, e: if conn is not None: conn.release(True) raise RequestTimeout(str(e))
def handle(self, sock, addr): p = HttpStream(SocketReader(sock)) path = p.path() if not path or path == "/": path = "index.html" if path.startswith("/"): path = path[1:] real_path = os.path.join(CURDIR, "static", path) if os.path.isdir(real_path): lines = ["<ul>"] for d in os.listdir(real_path): fpath = os.path.join(real_path, d) lines.append("<li><a href=" + d + ">" + d + "</a>") data = "".join(lines) resp = "".join(["HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n", "Content-Length:" + str(len(data)) + "\r\n", "Connection: close\r\n\r\n", data]) sock.sendall(resp) elif not os.path.exists(real_path): util.write_error(sock, 404, "Not found", real_path + " not found") else: ctype = mimetypes.guess_type(real_path)[0] if ctype.startswith('text') or 'html' in ctype: try: f = open(real_path, 'rb') data = f.read() resp = "".join(["HTTP/1.1 200 OK\r\n", "Content-Type: " + ctype + "\r\n", "Content-Length:" + str(len(data)) + "\r\n", "Connection: close\r\n\r\n", data]) sock.sendall(resp) finally: f.close() else: try: f = open(real_path, 'r') clen = int(os.fstat(f.fileno())[6]) # send headers sock.send("".join(["HTTP/1.1 200 OK\r\n", "Content-Type: " + ctype + "\r\n", "Content-Length:" + str(clen) + "\r\n", "Connection: close\r\n\r\n"])) if not sendfile: while True: data = f.read(4096) if not data: break sock.send(data) else: fileno = f.fileno() sockno = sock.fileno() sent = 0 offset = 0 nbytes = clen sent += sendfile(sockno, fileno, offset+sent, nbytes-sent) while sent != nbytes: sent += sendfile(sock.fileno(), fileno, offset+sent, nbytes-sent) finally: f.close()
def perform(self, request): """ perform the request. If an error happen it will first try to restart it """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to perform request: %s %s %s" % (request.host, request.method, request.path)) tries = 0 while True: conn = None try: # get or create a connection to the remote host conn = self.get_connection(request) # send headers msg = self.make_headers_string(request, conn.extra_headers) # send body if request.body is not None: chunked = request.is_chunked() if request.headers.iget('content-length') is None and \ not chunked: raise RequestError( "Can't determine content length and " + "Transfer-Encoding header is not chunked") # handle 100-Continue status # http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3 hdr_expect = request.headers.iget("expect") if hdr_expect is not None and \ hdr_expect.lower() == "100-continue": conn.send(msg) msg = None p = HttpStream(SocketReader(conn.socket()), kind=1, decompress=True) if p.status_code != 100: self.reset_request() if log.isEnabledFor(logging.DEBUG): log.debug("return response class") return self.response_class(conn, request, p) chunked = request.is_chunked() if log.isEnabledFor(logging.DEBUG): log.debug("send body (chunked: %s)" % chunked) if isinstance(request.body, (str, )): if msg is not None: conn.send(msg + to_bytestring(request.body), chunked) else: conn.send(to_bytestring(request.body), chunked) else: if msg is not None: conn.send(msg) if hasattr(request.body, 'read'): if hasattr(request.body, 'seek'): request.body.seek(0) conn.sendfile(request.body, chunked) else: conn.sendlines(request.body, chunked) if chunked: conn.send_chunk("") else: conn.send(msg) return self.get_response(request, conn) except socket.gaierror as e: if conn is not None: conn.release(True) raise RequestError(str(e)) except socket.timeout as e: if conn is not None: conn.release(True) raise RequestTimeout(str(e)) except socket.error as e: if log.isEnabledFor(logging.DEBUG): log.debug("socket error: %s" % str(e)) if conn is not None: conn.close() errors = (errno.EAGAIN, errno.EPIPE, errno.EBADF, errno.ECONNRESET) if e[0] not in errors or tries >= self.max_tries: raise RequestError("socket.error: %s" % str(e)) # should raised an exception in other cases request.maybe_rewind(msg=str(e)) except NoMoreData as e: if conn is not None: conn.release(True) request.maybe_rewind(msg=str(e)) if tries >= self.max_tries: raise except BadStatusLine: if conn is not None: conn.release(True) # should raised an exception in other cases request.maybe_rewind(msg="bad status line") if tries >= self.max_tries: raise except Exception: # unkown error log.debug("unhandled exception %s" % traceback.format_exc()) if conn is not None: conn.release(True) raise tries += 1 self._pool.backend_mod.sleep(self.wait_tries)
def read_output_data(newsocketconn, fromaddr): if useSSL: connstreamout = context.wrap_socket(newsocketconn, server_side=True) else: connstreamout = newsocketconn ### Output File Name outputfile = "Events_" + str(fromaddr[0]) + ".txt" global event_count outdata = headers = HostDetails = "" try: try: ### Read the json response using Socket Reader and split header and body r = SocketReader(connstreamout) p = HttpStream(r) headers = p.headers() print("headers: ", headers) bodydata = p.body_file().read() bodydata = bodydata.decode("utf-8") print("bodydata: ", bodydata) for eachHeader in headers.items(): if eachHeader[0] == 'Host' or eachHeader[0] == 'host': HostDetails = eachHeader[1] ### Read the json response and print the output print("\n") print("Server IP Address is ", fromaddr[0]) print("Server PORT number is ", fromaddr[1]) print("Listener IP is ", HostDetails) outdata = json.loads(bodydata) event_array = outdata['Events'] for event in event_array: print("EventType is ", event['EventType']) print("MessageId is ", event['MessageId']) if 'EventId' in event: print("EventId is ", event['EventId']) if 'EventTimestamp' in event: print("EventTimestamp is ", event['EventTimestamp']) if 'Severity' in event: print("Severity is ", event['Severity']) if 'Message' in event: print("Message is ", event['Message']) if 'MessageArgs' in event: print("MessageArgs is ", event['MessageArgs']) if 'Context' in outdata: print("Context is ", outdata['Context']) print("\n") except Exception as err: outdata = connstreamout.read() print("Data needs to read in normal Text format.") print(outdata) ### Check the context and send the status OK if context matches if outdata.get('Context', None) != ContextDetail: print("Context ({}) does not match with the server ({}).".format( outdata.get('Context', None), ContextDetail)) StatusCode = """HTTP/1.1 200 OK\r\n\r\n""" connstreamout.send(bytes(StatusCode, 'UTF-8')) try: if event_count.get(str(fromaddr[0])): event_count[str( fromaddr[0])] = event_count[str(fromaddr[0])] + 1 else: event_count[str(fromaddr[0])] = 1 print("Event Counter for Host %s is %s" % (str(fromaddr[0]), event_count[fromaddr[0]])) print("\n") fd = open(outputfile, "a") fd.write("Time:%s Count:%s\nHost IP:%s\nEvent Details:%s\n" % (time.ctime(), event_count[str( fromaddr[0])], str(fromaddr), outdata)) fd.close() except Exception as err: print(traceback.print_exc()) finally: connstreamout.shutdown(socket.SHUT_RDWR) connstreamout.close()
def test_continuation_header(): stream = io.BytesIO(b'GET /test HTTP/1.1\r\nX-Test: foo\r\n bar\r\n\r\n') hdr = HttpStream(stream).headers() assert hdr['X-Test'] == 'foo bar'
def handle(self, sock, addr): sock.setblocking(1) try: p = HttpStream(SocketReader(sock)) path = p.path() if not path or path == "/": path = "index.html" if path.startswith("/"): path = path[1:] real_path = os.path.join(self.conf['path'], path) if os.path.isdir(real_path): lines = ["<ul>"] for d in os.listdir(real_path): fpath = os.path.join(real_path, d) lines.append("<li><a href=" + d + ">" + d + "</a>") data = "".join(lines) resp = "".join(["HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n", "Content-Length:" + str(len(data)) + "\r\n", "Connection: close\r\n\r\n", data]) sock.sendall(resp) elif not os.path.exists(real_path): util.write_error(sock, 404, "Not found", real_path + " not found") else: ctype = mimetypes.guess_type(real_path)[0] fno = None try: fno = os.open(real_path, os.O_RDONLY|os.O_NONBLOCK) clen = int(os.fstat(fno)[6]) # send headers sock.send("".join(["HTTP/1.1 200 OK\r\n", "Content-Type: " + ctype + "\r\n", "Content-Length:" + str(clen) + "\r\n", "Connection: close\r\n\r\n"])) if not sendfile: while True: data = os.read(fno, 4096) if not data: break sock.send(data) else: offset = 0 nbytes = clen if nbytes > BLKSIZE: # Send file in at most 1GB blocks as some operating # systems can have problems with sending files in blocks # over 2GB. for m in range(0, nbytes, BLKSIZE): sendfile_all(fno, sock.fileno(), offset, min(nbytes, BLKSIZE)) offset += BLKSIZE nbytes -= BLKSIZE else: sendfile_all(fno, sock.fileno(), offset, nbytes) finally: if fno is not None: os.close(fno) finally: # make sure we close the socket try: sock.close() except: pass