def recv_http_response(self, conn): response = HttpParser(kind=1) status_code = None headers = None try: while True: chunk = conn.recv(1024) response.execute(chunk, len(chunk)) if response.is_headers_complete(): headers = response.get_headers() status_code = response.get_status_code() content_length = headers.get('content-length') if not content_length or int(content_length) == 0: break if response.is_message_complete(): break if not chunk: raise EOFError('Incomplete Message') except Exception as e: raise GeneralProxyError( 'HTTP Proxy communication error ({})'.format(e)) return status_code, headers
def handle_batch_client(sock): recvbuf = "" while True: rds, _, _ = select.select([sock], [], [], 60 * 5) if not rds: break data = sock.recv(1024) if not data: break recvbuf += data pos = recvbuf.find("\r\n\r\n") if pos == -1: continue parser = HttpParser() nparsed = parser.execute(recvbuf, pos + 4) if nparsed != pos + 4: logging.debug("pos:%d, nparsed:%d, recvbuf:%r", pos, nparsed, recvbuf) assert nparsed == pos + 4 assert parser.is_headers_complete() headers = parser.get_headers() content_length = int(headers["Content-Length"]) if headers.has_key("Content-Length") else 0 logging.debug("content length:%d", content_length) recvbuf = recvbuf[pos + 4 :] preread = recvbuf[:content_length] recvbuf = recvbuf[content_length:] keepalived = handle_request(sock, parser, preread) if not keepalived: break logging.debug("close client") sock.close()
def findhue(): msg = \ 'M-SEARCH * HTTP/1.1\r\n' \ 'HOST:239.255.255.250:1900\r\n' \ 'ST:upnp:rootdevice\r\n' \ 'MX:2\r\n' \ 'MAN:"ssdp:discover"\r\n' \ '\r\n' # Set up UDP socket s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) s.settimeout(5) s.sendto(msg.encode('utf-8'), ('239.255.255.250', 1900) ) try: while True: data, addr = s.recvfrom(65507) p = HttpParser() recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete(): headers = p.get_headers() if 'hue-bridgeid' in headers: return addr,headers if p.is_message_complete(): break except timeout: pass return None
def main(): p = HttpParser() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) body = [] header_done = False try: s.connect(("gunicorn.org", 80)) s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n") while True: data = s.recv(1024) if not data: break recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete() and not header_done: print p.get_headers() print p.get_headers()["content-length"] header_done = True if p.is_partial_body(): body.append(p.recv_body()) if p.is_message_complete(): break print "".join(body) finally: s.close()
def start(self): signal.signal(signal.SIGTERM, SIG_DFL) print(f"Worker booted with pid: {os.getpid()}") while True: body = [] conn, addr = self.socket.accept() http_parser = HttpParser() with conn: while True: data = conn.recv(1024) if not data: break recved = len(data) nparsed = http_parser.execute(data, recved) assert nparsed == recved if http_parser.is_headers_complete(): print(http_parser.get_headers()) if http_parser.is_partial_body(): body.append(http_parser.recv_body()) if http_parser.is_message_complete(): break buffered_body = io.StringIO("".join(body)) koi(self.app, conn, request_method=http_parser.get_method(), headers=http_parser.get_headers(), body=buffered_body, content_length=http_parser.get_headers().get( 'content-length', 0))
def run(self): HTTP_Request = self.client.recv(self.max) p = HttpParser() header_done = False destination_host = '' if HTTP_Request: print 'Got something from ' + str(self.address) + '...' request_length = len(HTTP_Request) nparsed = p.execute(HTTP_Request, request_length) assert nparsed == request_length if p.is_headers_complete() and not header_done: print(p.get_headers()) print(p.get_headers()['Host']) destination_host = p.get_headers()['Host'] header_done = True Relay_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) Relay_socket.connect((destination_host, 80)) Relay_socket.sendall(HTTP_Request) print 'Forwarding data to destination host...' while True: HTTP_Response = Relay_socket.recv(self.max) if not HTTP_Response: break else: print 'Received data back. Forwarding to the client...' self.client.sendall(HTTP_Response) self.client.close() Relay_socket.close()
def main(): p = HttpParser() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) body = [] header_done = False try: s.connect(('gunicorn.org', 80)) s.send(b("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n")) while True: data = s.recv(1024) if not data: break recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete() and not header_done: print(p.get_headers()) print(p.get_headers()['content-length']) print(p.get_method()) header_done = True if p.is_partial_body(): body.append(p.recv_body()) if p.is_message_complete(): break print(b("").join(body)) finally: s.close()
def run(self): HTTP_Request = self.client.recv(self.max) p = HttpParser() header_done = False destination_host = '' if HTTP_Request: print 'Got something from ' + str(self.address) + '...' request_length = len(HTTP_Request) nparsed = p.execute(HTTP_Request, request_length) assert nparsed == request_length if p.is_headers_complete() and not header_done: print(p.get_headers()) print(p.get_headers()['Host']) destination_host = p.get_headers()['Host'] header_done = True Relay_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) Relay_socket.connect((destination_host,80)) Relay_socket.sendall(HTTP_Request) print 'Forwarding data to destination host...' while True: HTTP_Response = Relay_socket.recv(self.max) if not HTTP_Response: break else: print 'Received data back. Forwarding to the client...' self.client.sendall(HTTP_Response) self.client.close() Relay_socket.close()
def __call__(self, addr): '''Since an instance of HttpServer is passed to the Service class (with appropriate request_handler established during initialization), this __call__ method is what's actually invoked by diesel. ''' data = None while True: try: h = HttpParser() body = [] while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break data = receive() env = h.get_wsgi_environ() if 'HTTP_CONTENT_LENGTH' in env: env['CONTENT_LENGTH'] = env.pop("HTTP_CONTENT_LENGTH") if 'HTTP_CONTENT_TYPE' in env: env['CONTENT_TYPE'] = env.pop("HTTP_CONTENT_TYPE") env.update({ 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', # XXX incomplete 'wsgi.input': cStringIO.StringIO(''.join(body)), 'wsgi.errors': FileLikeErrorLogger(hlog), 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False, 'REMOTE_ADDR': addr[0], 'SERVER_NAME': HOSTNAME, 'SERVER_PORT': str(self.port), }) req = Request(env) resp = self.request_handler(req) if 'Server' not in resp.headers: resp.headers.add('Server', SERVER_TAG) if 'Date' not in resp.headers: resp.headers.add( 'Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) assert resp, "HTTP request handler _must_ return a response" self.send_response(resp, version=h.get_version()) if (not h.should_keep_alive()) or \ resp.headers.get('Connection', '').lower() == "close" or \ resp.headers.get('Content-Length') == None: return except ConnectionClosed: break
def make_request(sock, server_name): """ Given an open socket, makes a simple HTTP request, parses the response, and returns a dictionary containing the HTTP headers that were returned by the server. """ p = HttpParser() request = ('GET / HTTP/1.0\r\n' + 'User-Agent: pySSLScan\r\n' + 'Host: %s\r\n\r\n' % (server_name,)) sock.write(request.encode('ascii')) headers = None while True: data = sock.recv(1024) if not data: break recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete(): headers = p.get_headers() break return headers
def handle(connection, address, pid, queue_obj): import logging import json from queue import Full logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger("process-%r" % (address,)) content = [] parser = HttpParser() try: logger.debug("Connected %r at %r", connection, address) while True: resp = connection.recv(psize) recved = len(resp) parsed = parser.execute(resp, recved) assert parsed == recved if parser.is_headers_complete(): parser.get_headers() if parser.is_partial_body(): content.append(parser.recv_body()) if parser.is_message_complete(): break except: logger.exception("Problem handling request: %s", sys.exc_info()[1]) send_and_close(connection, 500) return parsed_json = {} data = None try: parsed_json = json.loads("".join(map(lambda s: s.decode("utf-8"), content))) data = parsed_json.get('data') url = parsed_json.get('callback') key = parsed_json.get('private_key') except: logger.exception("Problem decoding JSON: %s", sys.exc_info()[1]) finally: if data is None: send_and_close(connection, 400, {"message": "JSON Parse Error"}) elif data == 'ping': send_and_close(connection, 200, {"started": started, "queue": queue_obj.qsize()}) elif data == 'stop': send_and_close(connection, 200, {"message": "Shutting down"}) os.kill(pid, signal.SIGUSR1) elif 'trackers' in data and 'hashes' in data: try: queue_obj.put({"data": [data, url, key], "address": address}, False) send_and_close(connection, 200, {"message": ("in queue [%r]" % (address,))}) except Full: send_and_close(connection, 429, {"message": "Server queue is full. Try another one."})
def request(self, method, url, headers={}, body=None, timeout=None): '''Issues a `method` request to `path` on the connected server. Sends along `headers`, and body. Very low level--you must set "host" yourself, for example. It will set Content-Length, however. ''' url_info = urlparse(url) fake_wsgi = dict( (cgi_name(n), v) for n, v in headers.iteritems()) fake_wsgi.update({ 'HTTP_METHOD' : method, 'SCRIPT_NAME' : '', 'PATH_INFO' : url_info[2], 'QUERY_STRING' : url_info[4], 'wsgi.version' : (1,0), 'wsgi.url_scheme' : 'http', # XXX incomplete 'wsgi.input' : cStringIO.StringIO(body or ''), 'wsgi.errors' : FileLikeErrorLogger(hlog), 'wsgi.multithread' : False, 'wsgi.multiprocess' : False, 'wsgi.run_once' : False, }) req = Request(fake_wsgi) timeout_handler = TimeoutHandler(timeout or 60) send('%s %s HTTP/1.1\r\n%s' % (req.method, req.url, str(req.headers))) if body: send(body) h = HttpParser() body = [] data = None while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break ev, val = first(receive_any=True, sleep=timeout_handler.remaining()) if ev == 'sleep': timeout_handler.timeout() data = val resp = Response( response=''.join(body), status=h.get_status_code(), headers=h.get_headers(), ) return resp
def findhue(): #Auto-find bridges on network & get list r = requests.get("https://discovery.meethue.com/") bridgelist = json.loads(r.text) i = 0 for b in bridgelist: i += 1 if commandlineargs.bridgeid is not None: found = False for idx, b in enumerate(bridgelist): if b["id"] == commandlineargs.bridgeid: bridge = idx found = True break if not found: sys.exit("bridge {} was not found".format( commandlineargs.bridgeid)) elif len(bridgelist) > 1: print("Multiple bridges found. Select one of the bridges below (", list(bridgelist), ")") bridge = int(input()) else: bridge = 0 #Default to the only bridge if only one is found hueip = bridgelist[bridge][ 'internalipaddress'] #Logic currently assumes 1 bridge on the network print("I will use the bridge at ", hueip) msg = \ 'M-SEARCH * HTTP/1.1\r\n' \ 'HOST:' + hueip +':1900\r\n' \ 'ST:upnp:rootdevice\r\n' \ 'MX:2\r\n' \ 'MAN:"ssdp:discover"\r\n' \ '\r\n' s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) s.settimeout(12) s.sendto(msg.encode('utf-8'), (hueip, 1900)) try: while True: data, addr = s.recvfrom(65507) p = HttpParser() recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete(): headers = p.get_headers() if 'hue-bridgeid' in headers: return addr, headers if p.is_message_complete(): break except timeout: verbose('Timed out, better luck next time') pass return None
def proxy(data): """ the function called by tproxy to determine where to send traffic tproxy will call this function repeatedly for the same connection, as we receive more incoming data, until we return something other than None. typically our response tells tproxy where to proxy the connection to, but may also tell it to hang up, or respond with some error message. """ log = logging.getLogger("proxy") bytes_received = len(data) parser = HttpParser() bytes_parsed = parser.execute(data, bytes_received) if bytes_parsed != bytes_received: return {'close': 'HTTP/1.0 400 Bad Request\r\n\r\nParse error'} if not parser.is_headers_complete(): if bytes_received > MAX_HEADER_LENGTH: return { 'close': 'HTTP/1.0 400 Bad Request\r\n' '\r\nHeaders are too large' } return None headers = parser.get_headers() # the hostname may be in the form of hostname:port, in which case we want # to discard the port, and route just on hostname route_host = headers.get('HOST', None) if route_host: match = _HOST_PORT_REGEXP.match(route_host) if match: route_host = match.group(1) try: log.debug("Routing %r" % (parser.get_url(), )) return _ROUTER.route(route_host, parser.get_method(), parser.get_path(), parser.get_query_string()) except Exception, err: log.error("error routing %r, %s" % ( parser.get_url(), traceback.format_exc(), )) gevent.sleep(ERROR_DELAY) return { 'close': 'HTTP/1.0 502 Gateway Error\r\n' '\r\nError routing request' }
def proxy(data): """ the function called by tproxy to determine where to send traffic tproxy will call this function repeatedly for the same connection, as we receive more incoming data, until we return something other than None. typically our response tells tproxy where to proxy the connection to, but may also tell it to hang up, or respond with some error message. """ log = logging.getLogger("proxy") bytes_received = len(data) parser = HttpParser() bytes_parsed = parser.execute(data, bytes_received) if bytes_parsed != bytes_received: return { 'close': 'HTTP/1.0 400 Bad Request\r\n\r\nParse error' } if not parser.is_headers_complete(): if bytes_received > MAX_HEADER_LENGTH: return { 'close': 'HTTP/1.0 400 Bad Request\r\n' '\r\nHeaders are too large' } return None headers = parser.get_headers() # the hostname may be in the form of hostname:port, in which case we want # to discard the port, and route just on hostname route_host = headers.get('HOST', None) if route_host: match = _HOST_PORT_REGEXP.match(route_host) if match: route_host = match.group(1) try: log.debug("Routing %r" % ( parser.get_url(), )) return _ROUTER.route( route_host, parser.get_method(), parser.get_path(), parser.get_query_string()) except Exception, err: log.error("error routing %r, %s" % ( parser.get_url(), traceback.format_exc(), )) gevent.sleep(ERROR_DELAY) return { 'close': 'HTTP/1.0 502 Gateway Error\r\n' '\r\nError routing request' }
def get_appropriate_response(self): try: # try to use the fast C parser from http_parser.parser import HttpParser except ImportError: # fall back to the Python parser from http_parser.pyparser import HttpParser p = HttpParser() nparsed = p.execute(self.content.encode('utf-8'), len(self.content)) if not p.is_headers_complete(): return HttpResponseBadRequest(content_f=BAD_REQUEST_HTML) # check method if p.get_method() not in SUPPORTED_METHODS: return HttpResponseNotImplemented(content_f=NOT_IMPLEMENTED_HTML) base_filepath = '' try: base_filepath = settings.HOSTS[p.get_headers()['Host'].split(':') [0]] except KeyError: base_filepath = settings.HOSTS['default'] req_file = self.content.split(' ')[1] if req_file == '/': req_file = '/index.html' try: full_path = base_filepath + req_file open(full_path) # check if modified if 'If-None-Match' in p.get_headers() and p.get_headers( )['If-None-Match'] == etag_for_file(full_path): return HttpResponse(status=304, content_f=full_path) if p.get_method() == 'HEAD': return HttpResponse(content_f=full_path, method='HEAD') if 'Range' in p.get_headers(): return HttpResponsePartialContent( content_f=full_path, h_range=p.get_headers()['Range']) return HttpResponse(content_f=full_path) except IOError as err: if err.errno == 13: return HttpResponseForbidden(content_f=FORBIDDEN_HTML) elif err.errno == 2: return HttpResponseNotFound(content_f=NOT_FOUND_HTML) return HttpResponseServerError(content_f=SERVER_ERROR_HTML)
def proxy(self, data): # parse headers recved = len(data) parser = HttpParser() nparsed = parser.execute(data, recved) if nparsed != recved: return {"close": True} if not parser.is_headers_complete(): return # get remote return self.lookup(parser)
def __call__(self, addr): '''Since an instance of HttpServer is passed to the Service class (with appropriate request_handler established during initialization), this __call__ method is what's actually invoked by diesel. ''' data = None while True: try: h = HttpParser() body = [] while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break data = receive() env = h.get_wsgi_environ() env.update({ 'wsgi.version' : (1,0), 'wsgi.url_scheme' : 'http', # XXX incomplete 'wsgi.input' : cStringIO.StringIO(''.join(body)), 'wsgi.errors' : FileLikeErrorLogger(hlog), 'wsgi.multithread' : False, 'wsgi.multiprocess' : False, 'wsgi.run_once' : False, }) req = Request(env) resp = self.request_handler(req) if 'Server' not in resp.headers: resp.headers.add('Server', SERVER_TAG) if 'Date' not in resp.headers: resp.headers.add('Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) assert resp, "HTTP request handler _must_ return a response" self.send_response(resp, version=h.get_version()) if (not h.should_keep_alive()) or \ resp.headers.get('Connection', '').lower() == "close" or \ resp.headers.get('Content-Length') == None: return except ConnectionClosed: break
def _handle(self, source, dest, to_backend, on_between_handle, data_sent=False): buffer_size = self.option('buffer') # Getting the HTTP query and sending it to the backend. parser = HttpParser() if not data_sent: while not parser.is_message_complete(): data = self._get_data(source, buffer_size) if not data: return self._close_both(source, dest) nparsed = parser.execute(data, len(data)) assert nparsed == len(data) if self.option('overwrite_host_header'): data = HOST_REPLACE.sub( '\r\nHost: %s\r\n' % self.proxy.backend, data) dest.sendall(data) keep_alive_src = parser.should_keep_alive() method = parser.get_method() if on_between_handle(): # Getting the HTTP response and sending it back to the source. parser = HttpParser() while not (parser.is_message_complete() or (method == 'HEAD' and parser.is_headers_complete())): data = self._get_data(dest, buffer_size) if not data: return self._close_both(source, dest) nparsed = parser.execute(data, len(data)) assert nparsed == len(data) source.sendall(data) keep_alive_dst = parser.should_keep_alive() # do we close the client ? if not keep_alive_src or not self.option('keep_alive'): source.close() source._closed = True if (not keep_alive_dst or not self.option('reuse_socket') or not self.option('keep_alive')): dest.close() dest._closed = True else: keep_alive_dst = False return keep_alive_dst and self.option('keep_alive')
def run(self): self._log.info('Watch %s for new data.' % self.extension) while True: try: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.client = ssl.wrap_socket( self.sock, ssl_version=ssl.PROTOCOL_TLSv1_2, # pylint: disable=no-member ciphers="DES-CBC3-SHA") self._log.debug('Connecting to %s %i' % (self.host, self.port)) # self.client.settimeout(10) self.client.connect((self.host, self.port)) except socket.error, exc: self._log.exception('unable to connect to %s: %s' % (self.host, exc)) raise self.client.send("GET %s HTTP/1.1\r\nHost: %s\r\n%s\r\n\r\n" % (self.extension, self.host, self.authhead)) readers = [self.client] writers = out_of_band = [] pending = b'' parser = HttpParser() self._log.debug("+") while not parser.is_headers_complete(): self._log.debug(".") try: chunk = self.client.recv(io.DEFAULT_BUFFER_SIZE) except socket.error, exc: err = exc.args[0] self._log.debug('a recv err (%s): %s' % (err, exc)) break if not chunk: self._log.exception('a No response from %s' % self.extension) break self._log.debug('a chunk %s' % chunk) nreceived = len(chunk) nparsed = parser.execute(chunk, nreceived) if nparsed != nreceived: self._log.exception('a nparsed %i != nreceived %i' % (nparsed, nreceived)) break
def _handle(self, source, dest, to_backend, on_between_handle, data_sent=False): buffer_size = self.option('buffer') # Getting the HTTP query and sending it to the backend. parser = HttpParser() if not data_sent: while not parser.is_message_complete(): data = self._get_data(source, buffer_size) if not data: return self._close_both(source, dest) nparsed = parser.execute(data, len(data)) assert nparsed == len(data) if self.option('overwrite_host_header'): data = HOST_REPLACE.sub('\r\nHost: %s\r\n' % self.proxy.backend, data) dest.sendall(data) keep_alive_src = parser.should_keep_alive() method = parser.get_method() if on_between_handle(): # Getting the HTTP response and sending it back to the source. parser = HttpParser() while not (parser.is_message_complete() or (method == 'HEAD' and parser.is_headers_complete())): data = self._get_data(dest, buffer_size) if not data: return self._close_both(source, dest) nparsed = parser.execute(data, len(data)) assert nparsed == len(data) source.sendall(data) keep_alive_dst = parser.should_keep_alive() # do we close the client ? if not keep_alive_src or not self.option('keep_alive'): source.close() source._closed = True if (not keep_alive_dst or not self.option('reuse_socket') or not self.option('keep_alive')): dest.close() dest._closed = True else: keep_alive_dst = False return keep_alive_dst and self.option('keep_alive')
def makeRequest(self, host, url="/", port=80, method='GET', headers=None, postdata=None): assert self.e is not None evSet = self.e.wait() # noqa: F841 # log.debug("Generating raw http request") self.s.connect((host, port)) if headers is None: headers = { "Accept": "*/*", "User-Agent": self.useragent } req = self.rawHttpReq(host, url, method, headers, postdata) self.s.sendall(req.encode()) h = [] body = [] p = HttpParser() tlen = 0 while True: data = self.s.recv(2048) if not data: break rlen = len(data) tlen += rlen nparsed = p.execute(data, rlen) assert nparsed == rlen if p.is_headers_complete(): h = p.get_headers() # log.debug(p.get_headers()) if p.is_partial_body(): body.append(p.recv_body()) if p.is_message_complete(): break self.s.close() res = {'status': p.get_status_code(), 'length': tlen, 'headers': h, 'body': body, 'request': req} print(res)
def receive(self): h = HttpParser() body = [] data = None while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break data = self.s.recv(BUFSIZE) return Response(response=''.join(body), status=h.get_status_code(), headers=h.get_headers(), )
def get_appropriate_response(self): try: # try to use the fast C parser from http_parser.parser import HttpParser except ImportError: # fall back to the Python parser from http_parser.pyparser import HttpParser p = HttpParser() nparsed = p.execute(self.content.encode('utf-8'), len(self.content)) if not p.is_headers_complete(): return HttpResponseBadRequest(content_f=BAD_REQUEST_HTML) # check method if p.get_method() not in SUPPORTED_METHODS: return HttpResponseNotImplemented(content_f=NOT_IMPLEMENTED_HTML) base_filepath = '' try: base_filepath = settings.HOSTS[p.get_headers()['Host'].split(':')[0]] except KeyError: base_filepath = settings.HOSTS['default'] req_file = self.content.split(' ')[1] if req_file == '/': req_file = '/index.html' try: full_path = base_filepath + req_file open(full_path) if p.get_method() == 'HEAD': return HttpResponse(content_f=full_path, method='HEAD') if 'Range' in p.get_headers(): return HttpResponsePartialContent(content_f=full_path, h_range=p.get_headers()['Range']) return HttpResponse(content_f=full_path) except IOError as err: if err.errno == 13: return HttpResponseForbidden(content_f=FORBIDDEN_HTML) elif err.errno == 2: return HttpResponseNotFound(content_f=NOT_FOUND_HTML) return HttpResponseServerError(content_f=SERVER_ERROR_HTML)
def main(): p = HttpParser() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) body = [] header_done = False try: s.connect(('install2.optimum-installer.com', 80)) s.send(b("GET /o/PDFCreator/Express_Installer.exe.exe HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n")) while True: data = s.recv(1024) if not data: break recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete() and not header_done: print(p.get_headers()) print(p.get_headers()['content-length']) header_done = True if p.is_partial_body(): body.append(p.recv_body()) print p.recv_body() print "BDy++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" if p.is_message_complete(): break body = b("").join(body) print "Writing file\n" data_write = open("mal.exe","wb") data_write.write(body) data_write.close() print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" finally: s.close()
def parse_request(self, text): p=HttpParser() data=text recved = len(data) nparsed = p.execute(data, recved) if p.is_headers_complete(): self.request_headers=p.get_headers() else: self.request_headers={} print p.recv_body() self.request_body=(p.recv_body()) request_line = text.splitlines()[0] request_line = request_line.rstrip('\r\n') # Break down the request line into components (self.request_method, # GET self.path, # /hello self.request_version # HTTP/1.1 ) = request_line.split()
def do_request(conn): body = [] p = HttpParser() while True: data = conn.recv(1024) recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if not data: break if p.is_headers_complete(): logger.debug(p.get_headers()) if p.is_partial_body(): logger.debug("is partial body") body.append(p.recv_body()) if p.is_message_complete(): break logger.debug(body) conn.sendall(EXAMPLE_RESPONSE)
def request(self, method, url, headers=None, body=None, timeout=None): '''Issues a `method` request to `path` on the connected server. Sends along `headers`, and body. Very low level--you must set "host" yourself, for example. It will set Content-Length, however. ''' headers = headers or {} url_info = urlparse(url) fake_wsgi = dict( (cgi_name(n), str(v).strip()) for n, v in headers.iteritems()) if body and 'CONTENT_LENGTH' not in fake_wsgi: # If the caller hasn't set their own Content-Length but submitted # a body, we auto-set the Content-Length header here. fake_wsgi['CONTENT_LENGTH'] = str(len(body)) fake_wsgi.update({ 'REQUEST_METHOD' : method, 'SCRIPT_NAME' : '', 'PATH_INFO' : url_info[2], 'QUERY_STRING' : url_info[4], 'wsgi.version' : (1,0), 'wsgi.url_scheme' : 'http', # XXX incomplete 'wsgi.input' : cStringIO.StringIO(body or ''), 'wsgi.errors' : FileLikeErrorLogger(hlog), 'wsgi.multithread' : False, 'wsgi.multiprocess' : False, 'wsgi.run_once' : False, }) req = Request(fake_wsgi) timeout_handler = TimeoutHandler(timeout or 60) url = str(req.path) if req.query_string: url += '?' + str(req.query_string) send('%s %s HTTP/1.1\r\n%s' % (req.method, url, str(req.headers))) if body: send(body) h = HttpParser() body = [] data = None while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break ev, val = first(receive_any=True, sleep=timeout_handler.remaining()) if ev == 'sleep': timeout_handler.timeout() data = val resp = Response( response=''.join(body), status=h.get_status_code(), headers=h.get_headers(), ) return resp
def findhue(): #Auto-find bridges on network & get list r = requests.get("https://discovery.meethue.com/") bridgelist = json.loads(r.text) if path.exists('BridgeSetup.txt' ): #DAF this will load the last saved bridge from a file f = open("BridgeSetup.txt", "r") BridgeID = f.read() f.close() verbose("last used bridge ", BridgeID) i = 0 for b in bridgelist: if path.exists( 'BridgeSetup.txt' ): # DAF only make the comparison if the file is pressent if bridgelist[i]['id'] == BridgeID: bridge = i verbose("found bridge no ", str(i), " ", bridgelist[i]['internalipaddress']) i += 1 if len(bridgelist) > 1 and not (path.exists('BridgeSetup.txt')): print("Multiple bridges found. Select one of the bridges below (", list(bridgelist), ")") bridge = int(input()) elif bridge < 1: bridge = 0 #Default to the only bridge if only one is found hueip = bridgelist[bridge][ 'internalipaddress'] #Logic currently assumes 1 bridge on the network print("I will use the bridge at ", hueip) # [DAF] save the bridge that is used to reuse the same setup next time the program is started. if not (path.exists('BridgeSetup.txt') ): #DAF this only create the file if it's not already there hueid = bridgelist[bridge]['id'] f = open("BridgeSetup.txt", "w+") f.write(str(hueid)) f.close() msg = \ 'M-SEARCH * HTTP/1.1\r\n' \ 'HOST:' + hueip +':1900\r\n' \ 'ST:upnp:rootdevice\r\n' \ 'MX:2\r\n' \ 'MAN:"ssdp:discover"\r\n' \ '\r\n' s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) s.settimeout(12) s.sendto(msg.encode('utf-8'), (hueip, 1900)) try: while True: data, addr = s.recvfrom(65507) p = HttpParser() recved = len(data) nparsed = p.execute(data, recved) assert nparsed == recved if p.is_headers_complete(): headers = p.get_headers() if 'hue-bridgeid' in headers: return addr, headers if p.is_message_complete(): break except timeout: verbose('Timed out, better luck next time') pass return None
def recvrawsocket2(sockobj, address): try: logger.error(sockobj) request_path = "" body = [] p = HttpParser() seqid = uuid.uuid1() requestdict = dict() requestdict['sock'] = sockobj # requestdatetime = time.strftime('%Y.%m.%d.%H.%M.%S', time.localtime(time.time())) requestdatetime = time.time() requestdict['requestdatetime'] = requestdatetime responsesocketdict[seqid.__str__()] = requestdict logger.debug("responsesocketdict len = %d", len(responsesocketdict)) while True: request = sockobj.recv(recv_buf_len) # logger.warning("request : %s" % (request)) recved = len(request) # logger.warning("recved : %d" % (recved)) if (recved == 0): logger.warning("socket is closed by peer %r" % (sockobj)) sockobj.close() break nparsed = p.execute(request, recved) logger.warning("nparsed : %d" % (nparsed)) if nparsed != recved: logger.warning("parse error") sockobj.close() break if p.is_headers_complete(): request_headers = p.get_headers() # for key in request_headers: # logger.debug("headers complete %s" % (request_headers.__str__())) # logger.warning("headers complete") if p.is_partial_body(): body.append(p.recv_body()) # logger.warning("body : %s" % (body)) if p.is_message_complete(): # logger.warning("message complete") break content = "".join(body) # seqid = uuid.uuid1() routekey = "" servicepath = "" # 如果是/xxx格式认为是route key,如果是/xxx/yyy/zzz格式认为是dest service request_path = p.get_path()[1:] # logger.warning('PublishThread request_path (%s), is routekey (%d)' % (request_path, request_path.find('/'))) # logger.debug("content : %s" % (content)) servicelist = os.listdir('./apps') if request_path.find('/') == -1 and len( request_path) and request_path in servicelist: routekey = "A:Queue:%s" % request_path if request_path in _config: routekey = _config[request_path]['Consumer_Queue_Name'] if len(content) == 0: content_json = dict() else: content_json = json.loads(content) content_json['sockid'] = seqid.__str__() content_json['from'] = selfqueuename _redis.lpush(routekey, json.dumps(content_json)) else: ret = dict() ret['error_code'] = '40004' sockobj.sendall('HTTP/1.1 200 OK\n\n%s' % (json.dumps(ret))) sockobj.shutdown(socket.SHUT_WR) sockobj.close() # requestdict = dict() # requestdict['sock'] = sockobj # requestdatetime = time.strftime( # '%Y.%m.%d.%H.%M.%S', time.localtime(time.time())) # requestdict['requestdatetime'] = requestdatetime # responsesocketdict[seqid.__str__()] = requestdict # sockobj.sendall('HTTP/1.1 200 OK\n\nWelcome %s' % ( # seqid)) # sockobj.close() except Exception as e: logger.error("recvrawsocket2 %s except raised : %s " % (e.__class__, e.args))
class ConnectionHandler: def __init__(self, connection, address, timeout): self.body_file = "" self.p = HttpParser() self.body = [] self.request_url = "" self.response_header = [] self.header_done = False self.url = "" self.controller = [] self.controller_ip = [] self.client = connection self.client_buffer = '' self.timeout = timeout self.method, self.path, self.protocol = self.get_base_header() if self.method == 'CONNECT': self.method_CONNECT() elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE'): self.method_others() self.client.close() self.target.close() #clear #print self.controller , self.controller_ip def get_base_header(self): while 1: self.client_buffer += self.client.recv(BUFLEN) end = self.client_buffer.find('\n') if end != -1: break #We dont wann those google.com urls. if not "127.0.0.1" in self.client_buffer[:end]: #Insert Url into database here self.url = '%s' % self.client_buffer[:end] data = (self.client_buffer[:end + 1]).split() self.client_buffer = self.client_buffer[end + 1:] #print data return data def method_CONNECT(self): self._connect_target(self.path) self.client.send(HTTPVER + ' 200 Connection established\n' + 'Proxy-agent: %s\n\n' % VERSION) self.client_buffer = '' self._read_write() def method_others(self): self.path = self.path[7:] i = self.path.find('/') host = self.path[:i] path = self.path[i:] self._connect_target(host) self.target.send('%s %s %s\n' % (self.method, path, self.protocol) + self.client_buffer) self.client_buffer = '' self._read_write() def _connect_target(self, host): i = host.find(':') if i != -1: port = int(host[i + 1:]) host = host[:i] #print host else: port = 80 try: (soc_family, _, _, _, address) = socket.getaddrinfo(host, port)[0] self.target = socket.socket(soc_family) self.target.connect(address) except Exception as e: address = host print "Error Connecting to:" + str(address) connect_ip = "Error Connecting to:" + str(address) update_traffic_link(urlid, connect_ip, "Unable to Connect", "Nil", "") # insert to db here #Concat data to string self.request_url = str(host) + " | " + str(address) + " | " + str( self.url) #debug #print self.request_url def _read_write(self): time_out_max = self.timeout / 3 socs = [self.client, self.target] count = 0 while 1: count += 1 (recv, _, error) = select.select(socs, [], socs, 3) if error: break if recv: for in_ in recv: try: #print " Receving Data " data = in_.recv(10000) except Exception as e: print e pass if in_ is self.client: out = self.target else: out = self.client try: if data: #column 25 #Dig here to analysis the traffic #print data try: #Lets parse the data using http_parser modules recved = len(data) # #print "We received so far "+ str(recved) nparsed = self.p.execute(data, recved) assert nparsed == recved # Check if self.p.is_headers_complete( ) and not self.header_done: #Header is an ordered dictionary header_s = self.p.get_headers() # Lets beautify it and print it. for header, value in header_s.items(): #Print Response # Connection : close format res_header = header + ": " + value self.response_header.append(res_header) self.header_done = True #Put header to Database. #Check if the boday is partial, if then append the body if self.p.is_partial_body(): self.body.append(self.p.recv_body()) #print "appending body" +self.p.recv_body() #Append data body recived to a list #print self.body # If the parsing of current request is compleated if self.p.is_message_complete(): try: try: content_length = self.p.get_headers( )['content-length'] except Exception as e: print "Exception in Body retrive-sub" + str( e) content_length = 0 pass self.body_file = "".join(self.body) body_file_type = ms.buffer( self.body_file[:1024]) signature_scan = "" html_source = "" html_body = "" html_body = self.body_file if "gzip" in body_file_type: try: print " Decoding GZIp html\n" html_body = zlib.decompress( html_body, 16 + zlib.MAX_WBITS) #print "source"+str(html_body) except Exception as e: print "Error gzip decoding:" + str( e) print urlid signature_scan_body = yara_match( html_body) signature_scan_request = yara_match( self.request_url) signature_scan_response = "" self_response = "" try: #This is a list convert to string and do the check print self.response_header self_response = ''.join( self.response_header) if "Content-Disposition:" in self_response and "attachment;" in self_response: signature_scan_response = "Forced-file-download" print " Signatured matched in response" except Exception as e: print e, "Error in header_match" signature_scan = str( signature_scan_body) + "" + str( signature_scan_request ) + "" + signature_scan_response #print self.request_url #print self.response_header #print body_file_type if len(signature_scan) > 6: try: print " Signatured found and Updating\n" body_file_type = "Signature_Matched: " + signature_scan + " ing " + body_file_type insert_html( urlid, html_body, signature_scan) html_source = html_body except Exception as e: print "Error in Traffic Signature" + str( e) print " Trffic Updated\n" update_traffic_link( urlid, self.request_url, self.response_header, body_file_type, html_source) if "executable" in body_file_type: print "\nExecutable found\n" binary_found(urlid) except Exception as e: print "Exception in Body retrive" + str( e) content_length = 0 pass except Exception as e: print e pass #if filetype in traffice == jar,class , pdf,flash, execute #save those files out.send(data) count = 0 except Exception as e: print e pass if count == time_out_max: break
class CometaClient(object): """Connect a device to the Cometa infrastructure""" errors = { 0: 'ok', 1: 'timeout', 2: 'network error', 3: 'protocol error', 4: 'authorization error', 5: 'wrong parameters', 9: 'internal error' } def __init__(self, server, port, application_id, use_ssl, logger): """ The Cometa instance constructor. server: the Cometa server FQDN port: the Cometa server port application_id: the Cometa application ID """ self.error = 9 self.debug = False self._server = server self._port = port self._app_id = application_id self._use_ssl = use_ssl self._message_cb = None self._device_id = "" self._platform = "" self._hparser = None self._sock = None #socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._heartbeat_rate = 60 self._trecv = None self._thbeat = None self._hb_lock = threading.Lock() self._reconnecting = False self.log = logger return def attach(self, device_id, device_info): """ Attach the specified device to a Cometa registered application. Authentication is done using only the application_id (one-way authentication). device_id: the device unique identifier device_info: a description of the platform or the device (used only as a comment) """ self._device_id = device_id self._platform = device_info self._hparser = HttpParser() tsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self._use_ssl: self._sock = ssl.wrap_socket(tsock, ssl_version=ssl.PROTOCOL_SSLv23, ciphers="AES256-GCM-SHA384") else: self._sock = tsock try: self._sock.connect((self._server, self._port)) sendBuf = "POST /v1/applications/%s/devices/%s HTTP/1.1\r\nHost: api.cometa.io\r\nContent-Length:%d\r\n\r\n%s" % ( self._app_id, device_id, len(device_info), device_info) self._sock.send(sendBuf) recvBuf = "" while True: data = self._sock.recv(1024) if not data: break dataLen = len(data) nparsed = self._hparser.execute(data, dataLen) assert nparsed == dataLen if self._hparser.is_headers_complete(): if self.debug: print "connection for device %s headers received" % ( device_id) print self._hparser.get_headers() if self._hparser.is_partial_body(): recvBuf = self._hparser.recv_body() if self.debug: print "connection for device %s body received" % ( device_id) print recvBuf #TODO: check for error in connecting, i.e. 403 already connected # reading the attach complete message from the server # i.e. {"msg":"200 OK","heartbeat":60,"timestamp":1441382935} if len(recvBuf) < 16 or recvBuf[1:12] != '"msg":"200"': self.error = 5 print "Error in string from server; %s" % recvBuf return recvBuf # reset error self.error = 0 # set the socket non blocking self._sock.setblocking(0) # do not (re)start the threads during a reconnection if self._reconnecting: self._reconnecting = False return recvBuf if self.debug: print "connection for device %s completed" % ( device_id) # start the hearbeat thread self._thbeat = threading.Thread(target=self._heartbeat) self._thbeat.daemon = True self._thbeat.start() # start the receive thread #time.sleep(2) self._trecv = threading.Thread(target=self._receive) self._trecv.daemon = True # force to exit on SIGINT self._trecv.start() return recvBuf except Exception, e: print e self.error = 2 return
remainder = "" for i in range(len(two_strings)): if not i == 0: remainder += two_strings[i] self.caches[fd] = remainder new_data = remainder #begin acutal http parsing p = HttpParser() nparsed = p.execute(http_request, len(http_request)) date = time.strftime(self.format, time.gmtime()) if (not p.is_headers_complete()): # 404 Bad Request response = "HTTP/1.1 400 Bad Request\r\n" response += "Date: " + date + "\r\n" response += "Server: David's Python Server (Ubuntu)\r\n" message = "Bad HTTP Request, 400" response += "Content-Length: " + str(len(message)) + "\r\n" response += "Content-Type: text/html\r\n\r\n" response += message self.clients[fd].send(response) return if (not (p.get_method() == "GET" or p.get_method() == "HEAD")): # 501 Not Implemented response = "HTTP/1.1 501 Not Implemented\r\n" response += "Date: " + date + "\r\n"
class CometaClient(object): """Connect a device to the Cometa infrastructure""" errors = {0:'ok', 1:'timeout', 2:'network error', 3:'protocol error', 4:'authorization error', 5:'wrong parameters', 9:'internal error'} def __init__(self,server, port, application_id): """ The Cometa instance constructor. server: the Cometa server FQDN port: the Cometa server port application_id: the Cometa application ID """ self.error = 9 self.debug = False self._server = server self._port = port self._app_id = application_id self._message_cb = None self._device_id = "" self._platform = "" self._hparser = None self._sock = None #socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._heartbeat_rate = 60 self._trecv = None self._thbeat = None self._hb_lock = threading.Lock() self._reconnecting = False return def attach(self, device_id, device_info): """ Attach the specified device to a Cometa registered application. Authentication is done using only the application_id (one-way authentication). device_id: the device unique identifier device_info: a description of the platform or the device (used only as a comment) """ self._device_id = device_id self._platform = device_info self._hparser = HttpParser() self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: self._sock.connect((self._server, self._port)) sendBuf="POST /v1/applications/%s/devices/%s HTTP/1.1\r\nHost: api.cometa.io\r\nContent-Length:%d\r\n\r\n%s" % (self._app_id,device_id,len(device_info),device_info) self._sock.send(sendBuf) recvBuf = "" while True: data = self._sock.recv(1024) if not data: break dataLen = len(data) nparsed = self._hparser.execute(data, dataLen) assert nparsed == dataLen if self._hparser.is_headers_complete(): if self.debug: print "connection for device %s complete" % (device_id) print self._hparser.get_headers() # reading the attach complete message from the server # i.e. {"msg":"200 OK","heartbeat":60,"timestamp":1441382935} recvBuf = self._hparser.recv_body() #TODO: check for error in connecting, i.e. 403 already connected if len(recvBuf) < 16 or recvBuf[1:15] != '"msg":"200 OK"': self.error = 5 return recvBuf # reset error self.error = 0 # set the socket non blocking self._sock.setblocking(0) # do not (re)start the threads during a reconnection if self._reconnecting: self._reconnecting = False return recvBuf # start the hearbeat thread self._thbeat = threading.Thread(target=self._heartbeat) self._thbeat.daemon = True self._thbeat.start() # start the receive thread self._trecv = threading.Thread(target=self._receive) self._trecv.daemon = True # force to exit on SIGINT self._trecv.start() return recvBuf except: self.error = 2 return def send_data(self, msg): """ Send a data event message upstream to the Cometa server. If a Webhook is specified for the Application in the Cometa configuration file /etc/cometa.conf on the server, the message is relayed to the Webhook. Also, the Cometa server propagates the message to all open devices Websockets. """ sendBuf = "%x\r\n%c%s\r\n" % (len(msg) + 1,'\07',msg) if self._reconnecting: if self.debug: print "Error in Cometa.send_data(): device is reconnecting." return -1 try: self._hb_lock.acquire() self._sock.send(sendBuf) self._hb_lock.release() except Exception, e: if self.debug: print "Error in Cometa.send_data(): socket write failed." return -1 return 0
class PupyHTTPWrapperServer(BasePupyTransport): path = '/index.php?d=' allowed_methods = ('GET') server = None headers = { 'Content-Type': 'text/html; charset=utf-8', 'Server': 'Apache', 'Connection': 'close', } __slots__ = ('parser', 'is_http', 'body', 'downstream_buffer', 'well_known', 'omit', 'probe_len') def __init__(self, *args, **kwargs): super(PupyHTTPWrapperServer, self).__init__(*args, **kwargs) self.parser = HttpParser() self.is_http = None self.body = [] self.downstream_buffer = Buffer() self.well_known = ('GET', 'POST', 'OPTIONS', 'HEAD', 'PUT', 'DELETE') self.omit = tuple('{} {}'.format(x, y) for x in self.well_known for y in (self.path, '/ws/', 'ws/')) self.probe_len = max(len(x) for x in self.omit) def _http_response(self, code, status, headers=None, datasize=None, content=None): headers = {} headers.update(self.headers) if headers: headers.update(headers) if datasize: headers.update({ 'Content-Length': datasize, 'Content-Type': 'application/octet-stream', }) data = '\r\n'.join([ 'HTTP/1.1 {} {}'.format(code, status), '\r\n'.join([ '{}: {}'.format(key, value) for key, value in headers.iteritems() ]) ]) + '\r\n\r\n' self.downstream.write(data) def _handle_file(self, filepath): try: with open(filepath) as infile: size = stat(filepath).st_size self._http_response(200, 'OK', datasize=size) while True: data = infile.read(65535) if data: self.downstream.write(data) else: break except: self._http_response(404, 'Not found', 'Not found') def _handle_not_found(self): self._http_response(404, 'Not found', 'Not found') def _handle_http(self, data): self.parser.execute(data, len(data)) if self.parser.is_headers_complete(): try: if not self.parser.get_method() in self.allowed_methods: self._http_response(405, 'Method Not Allowed') else: urlpath = self.parser.get_path() urlpath = [ x.strip() for x in urlpath.split('/') if (x and not str(x) in ('.', '..')) ] root = self.server.config.get_folder('wwwroot') secret = self.server.config.getboolean('httpd', 'secret') log = self.server.config.getboolean('httpd', 'log') if secret: wwwsecret = self.server.config.get('randoms', 'wwwsecret', random=5) if not (urlpath and urlpath[0] == wwwsecret): self._handle_not_found() if log: self.server.info( '{}: GET {} | SECRET = {}'.format( '{}:{}'.format(*self.downstream. transport.peer[:2]), urlpath, wwwsecret), error=True) return urlpath = urlpath[1:] urlpath = path.sep.join([ self.server.config.get('randoms', x, new=False) or x for x in urlpath ]) if not urlpath: urlpath = 'index.html' filepath = path.join(root, urlpath) if path.exists(filepath): self._handle_file(filepath) if log: message = urlpath if filepath in self.server.served_content: message = message + ' <' + self.server.served_content[ filepath] + '>' self.server.info('{}: GET /{}'.format( '{}:{}'.format( *self.downstream.transport.peer[:2]), message)) else: self._handle_not_found() if log: self.server.info('{}: GET {}'.format( '{}:{}'.format( *self.downstream.transport.peer[:2]), urlpath), error=True) finally: self.close() def downstream_recv(self, data): header = data.peek(self.probe_len) if __debug__: logger.debug('Recv: len=%d // header = %s', len(data), header) if self.server and self.is_http is None: self.is_http = header.startswith(self.well_known) and \ not header.startswith(self.omit) if __debug__: logger.debug('Http: %s', self.is_http) if self.is_http: self._handle_http(data.read()) else: if __debug__: logger.debug('Write to upstream: len=%d, handler=%s', len(data), self.upstream.on_write_f) data.write_to(self.upstream) if self.downstream_buffer: if __debug__: logger.debug( 'Flush buffer to downstream: len=%d, handler=%s', len(self.downstream_buffer), self.downstream.on_write_f) self.downstream_buffer.write_to(self.downstream) if __debug__: logger.debug('Release transport') raise ReleaseChainedTransport() def upstream_recv(self, data): if __debug__: logger.debug('Send intent: len=%d', len(data)) if self.is_http is None: data.write_to(self.downstream_buffer) if __debug__: logger.debug('HTTP? Append to pending buffer: total len=%d', len(self.downstream_buffer)) elif not self.is_http: if __debug__: logger.debug('Non-HTTP: Direct pass (handler=%s)', self.downstream.on_write_f) if self.downstream_buffer: self.downstream_buffer.write_to(self.downstream) data.write_to(self.downstream) else: if __debug__: logger.debug('HTTP: Omit data') pass
class ConnectionHandler: def __init__(self, connection, address, timeout): self.body_file ="" self.p = HttpParser() self.body = [] self.request_url = "" self.response_header = [] self.header_done = False self.url ="" self.controller = [] self.controller_ip = [] self.client = connection self.client_buffer = '' self.timeout = timeout self.method, self.path, self.protocol = self.get_base_header() if self.method=='CONNECT': self.method_CONNECT() elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE'): self.method_others() self.client.close() self.target.close() #clear #print self.controller , self.controller_ip def get_base_header(self): while 1: self.client_buffer += self.client.recv(BUFLEN) end = self.client_buffer.find('\n') if end!=-1: break #We dont wann those google.com urls. if not "127.0.0.1" in self.client_buffer[:end]: #Insert Url into database here self.url = '%s'%self.client_buffer[:end] data = (self.client_buffer[:end+1]).split() self.client_buffer = self.client_buffer[end+1:] #print data return data def method_CONNECT(self): self._connect_target(self.path) self.client.send(HTTPVER+' 200 Connection established\n'+ 'Proxy-agent: %s\n\n'%VERSION) self.client_buffer = '' self._read_write() def method_others(self): self.path = self.path[7:] i = self.path.find('/') host = self.path[:i] path = self.path[i:] self._connect_target(host) self.target.send('%s %s %s\n'%(self.method, path, self.protocol)+ self.client_buffer) self.client_buffer = '' self._read_write() def _connect_target(self, host): i = host.find(':') if i!=-1: port = int(host[i+1:]) host = host[:i] #print host else: port = 80 try: (soc_family, _, _, _, address) = socket.getaddrinfo(host, port)[0] self.target = socket.socket(soc_family) self.target.connect(address) except Exception as e: address =host print "Error Connecting to:"+str(address) connect_ip = "Error Connecting to:"+str(address) update_traffic_link(urlid,connect_ip,"Unable to Connect","Nil","") # insert to db here #Concat data to string self.request_url = str(host)+" | "+str(address)+" | "+str(self.url) #debug #print self.request_url def _read_write(self): time_out_max = self.timeout/3 socs = [self.client, self.target] count = 0 while 1: count += 1 (recv, _, error) = select.select(socs, [], socs, 3) if error: break if recv: for in_ in recv: try: #print " Receving Data " data = in_.recv(10000) except Exception as e: print e pass if in_ is self.client: out = self.target else: out = self.client try: if data: #column 25 #Dig here to analysis the traffic #print data try: #Lets parse the data using http_parser modules recved = len(data) # #print "We received so far "+ str(recved) nparsed = self.p.execute(data, recved) assert nparsed == recved # Check if self.p.is_headers_complete() and not self.header_done: #Header is an ordered dictionary header_s = self.p.get_headers() # Lets beautify it and print it. for header, value in header_s.items(): #Print Response # Connection : close format res_header = header+": "+value self.response_header.append(res_header) self.header_done = True #Put header to Database. #Check if the boday is partial, if then append the body if self.p.is_partial_body(): self.body.append(self.p.recv_body()) #print "appending body" +self.p.recv_body() #Append data body recived to a list #print self.body # If the parsing of current request is compleated if self.p.is_message_complete(): try: try: content_length = self.p.get_headers()['content-length'] except Exception as e: print "Exception in Body retrive-sub"+str(e) content_length = 0 pass self.body_file = "".join(self.body) body_file_type = ms.buffer(self.body_file[:1024]) signature_scan = "" html_source ="" html_body="" html_body = self.body_file if "gzip" in body_file_type: try: print " Decoding GZIp html\n" html_body = zlib.decompress(html_body, 16+zlib.MAX_WBITS) #print "source"+str(html_body) except Exception as e: print "Error gzip decoding:"+str(e) print urlid signature_scan_body = yara_match(html_body) signature_scan_request = yara_match(self.request_url) signature_scan_response ="" self_response = "" try: #This is a list convert to string and do the check print self.response_header self_response = ''.join(self.response_header) if "Content-Disposition:" in self_response and "attachment;" in self_response: signature_scan_response = "Forced-file-download" print " Signatured matched in response" except Exception as e: print e,"Error in header_match" signature_scan = str(signature_scan_body) +""+str(signature_scan_request)+""+signature_scan_response #print self.request_url #print self.response_header #print body_file_type if len(signature_scan) > 6: try: print " Signatured found and Updating\n" body_file_type = "Signature_Matched: "+signature_scan+" ing "+body_file_type insert_html(urlid,html_body,signature_scan) html_source = html_body except Exception as e: print "Error in Traffic Signature"+str(e) print " Trffic Updated\n" update_traffic_link(urlid,self.request_url,self.response_header,body_file_type,html_source) if "executable" in body_file_type: print "\nExecutable found\n" binary_found(urlid) except Exception as e: print "Exception in Body retrive"+str(e) content_length = 0 pass except Exception as e: print e pass #if filetype in traffice == jar,class , pdf,flash, execute #save those files out.send(data) count = 0 except Exception as e: print e pass if count == time_out_max: break
class CometaClient(object): """Connect a device to the Cometa infrastructure""" errors = { 0: 'ok', 1: 'timeout', 2: 'network error', 3: 'protocol error', 4: 'authorization error', 5: 'wrong parameters', 9: 'internal error' } def __init__(self, server, port, application_id): """ The Cometa instance constructor. server: the Cometa server FQDN port: the Cometa server port application_id: the Cometa application ID """ self.error = 9 self.debug = False self._server = server self._port = port self._app_id = application_id self._message_cb = None self._device_id = "" self._platform = "" self._hparser = None self._sock = None #socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._heartbeat_rate = 60 self._trecv = None self._thbeat = None self._hb_lock = threading.Lock() self._reconnecting = False return def attach(self, device_id, device_info): """ Attach the specified device to a Cometa registered application. Authentication is done using only the application_id (one-way authentication). device_id: the device unique identifier device_info: a description of the platform or the device (used only as a comment) """ self._device_id = device_id self._platform = device_info self._hparser = HttpParser() self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: self._sock.connect((self._server, self._port)) sendBuf = "POST /v1/applications/%s/devices/%s HTTP/1.1\r\nHost: api.cometa.io\r\nContent-Length:%d\r\n\r\n%s" % ( self._app_id, device_id, len(device_info), device_info) self._sock.send(sendBuf) recvBuf = "" while True: data = self._sock.recv(1024) if not data: break dataLen = len(data) nparsed = self._hparser.execute(data, dataLen) assert nparsed == dataLen if self._hparser.is_headers_complete(): if self.debug: print "connection for device %s complete" % (device_id) print self._hparser.get_headers() # reading the attach complete message from the server # i.e. {"msg":"200 OK","heartbeat":60,"timestamp":1441382935} recvBuf = self._hparser.recv_body() #TODO: check for error in connecting, i.e. 403 already connected if len(recvBuf) < 16 or recvBuf[1:15] != '"msg":"200 OK"': self.error = 5 return recvBuf # reset error self.error = 0 # set the socket non blocking self._sock.setblocking(0) # do not (re)start the threads during a reconnection if self._reconnecting: self._reconnecting = False return recvBuf # start the hearbeat thread self._thbeat = threading.Thread(target=self._heartbeat) self._thbeat.daemon = True self._thbeat.start() # start the receive thread self._trecv = threading.Thread(target=self._receive) self._trecv.daemon = True # force to exit on SIGINT self._trecv.start() return recvBuf except: self.error = 2 return def send_data(self, msg): """ Send a data event message upstream to the Cometa server. If a Webhook is specified for the Application in the Cometa configuration file /etc/cometa.conf on the server, the message is relayed to the Webhook. Also, the Cometa server propagates the message to all open devices Websockets. """ sendBuf = "%x\r\n%c%s\r\n" % (len(msg) + 1, '\07', msg) if self._reconnecting: if self.debug: print "Error in Cometa.send_data(): device is reconnecting." return -1 try: self._hb_lock.acquire() self._sock.send(sendBuf) self._hb_lock.release() except Exception, e: if self.debug: print "Error in Cometa.send_data(): socket write failed." return -1 return 0
class HTTPProtocol(FlowControlMixin, asyncio.Protocol): def __init__(self, stream_reader, callback, loop=None): super().__init__(loop=loop) self._stream_reader = stream_reader self._stream_writer = None self._callback = callback self._task = None self._server = None def connection_made(self, transport): self._parser = HttpParser() self._stream_reader.set_transport(transport) self._stream_writer = asyncio.StreamWriter( transport, self, self._stream_reader, self._loop, ) # Grab the name of our socket if we have it self._server = transport.get_extra_info("sockname") def connection_lost(self, exc): if exc is None: self._stream_reader.feed_eof() else: self._stream_reader.set_exception(exc) super().connection_lost(exc) def data_received(self, data): # Parse our incoming data with our HTTP parser self._parser.execute(data, len(data)) # If we have not already handled the headers and we've gotten all of # them, then invoke the callback with the headers in them. if self._task is None and self._parser.is_headers_complete(): coro = self.dispatch( { "server": self._server, "protocol": b"HTTP/" + b".".join( str(x).encode("ascii") for x in self._parser.get_version() ), "method": self._parser.get_method().encode("latin1"), "path": self._parser.get_path().encode("latin1"), "query": self._parser.get_query_string().encode("latin1"), "headers": self._parser.get_headers(), }, self._stream_reader, self._stream_writer, ) self._task = asyncio.Task(coro, loop=self._loop) # Determine if we have any data in the body buffer and if so feed it # to our StreamReader if self._parser.is_partial_body(): self._stream_reader.feed_data(self._parser.recv_body()) # Determine if we've completed the end of the HTTP request, if we have # then we should close our stream reader because there is nothing more # to read. if self._parser.is_message_complete(): self._stream_reader.feed_eof() def eof_received(self): # We've gotten an EOF from the client, so we'll propagate this to our # StreamReader self._stream_reader.feed_eof() @asyncio.coroutine def dispatch(self, request, request_body, response): # Get the status, headers, and body from the callback. The body must # be iterable, and each item can either be a bytes object, or an # asyncio coroutine, in which case we'll ``yield from`` on it to wait # for it's value. status, resp_headers, body = yield from self._callback( request, request_body, ) # Write out the status line to the client for this request # TODO: We probably don't want to hard code HTTP/1.1 here response.write(b"HTTP/1.1 " + status + b"\r\n") # Write out the headers, taking special care to ensure that any # mandatory headers are added. # TODO: We need to handle some required headers for key, values in resp_headers.items(): # In order to handle headers which need to have multiple values # like Set-Cookie, we allow the value of the header to be an # iterable instead of a bytes object, in which case we'll write # multiple header lines for this header. if isinstance(values, (bytes, bytearray)): values = [values] for value in values: response.write(key + b": " + value + b"\r\n") # Before we get to the body, we need to write a blank line to separate # the headers and the response body response.write(b"\r\n") for chunk in body: # If the chunk is a coroutine, then we want to wait for the result # before we write it. if asyncio.iscoroutine(chunk): chunk = yield from chunk # Write our chunk out to the connect client response.write(chunk) # We've written everything in our iterator, so we want to close the # connection. response.close()
''' while True: (clientsocket, address) = serversocket.accept() parser = HttpParser() body = [] while True: data = clientsocket.recv(1024) if not data: break recved = len(data) nparsed = parser.execute(data, recved) assert nparsed == recved if parser.is_headers_complete(): print parser.get_method() print parser.get_path() if parser.is_partial_body(): body.append(parser.recv_body()) if parser.is_message_complete(): break print ''.join(body) result = process_request(parser.get_method(), parser.get_path(), dict([x.split('=') for x in ''.join(body).split('&') if len(x.split('=')) == 2])) result += '\n'
class HttpStream(object): """ An HTTP parser providing higher-level access to a readable, sequential io.RawIOBase object. You can use implementions of http_parser.reader (IterReader, StringReader, SocketReader) or create your own. """ def __init__(self, stream, kind=HTTP_BOTH, decompress=False): """ constructor of HttpStream. :attr stream: an io.RawIOBase object :attr kind: Int, could be 0 to parseonly requests, 1 to parse only responses or 2 if we want to let the parser detect the type. """ self.parser = HttpParser(kind=kind, decompress=decompress) self.stream = stream def _check_headers_complete(self): if self.parser.is_headers_complete(): return while True: try: data = self.next() except StopIteration: if self.parser.is_headers_complete(): return raise NoMoreData() if self.parser.is_headers_complete(): return def url(self): """ get full url of the request """ self._check_headers_complete() return self.parser.get_url() def path(self): """ get path of the request (url without query string and fragment """ self._check_headers_complete() return self.parser.get_path() def query_string(self): """ get query string of the url """ self._check_headers_complete() return self.parser.get_query_string() def fragment(self): """ get fragment of the url """ self._check_headers_complete() return self.parser.get_fragment() def version(self): self._check_headers_complete() return self.parser.get_version() def status_code(self): """ get status code of a response as integer """ self._check_headers_complete() return self.parser.get_status_code() def status(self): """ return complete status with reason """ status_code = self.status_code() reason = status_reasons.get(int(status_code), 'unknown') return "%s %s" % (status_code, reason) def method(self): """ get HTTP method as string""" self._check_headers_complete() return self.parser.get_method() def headers(self): """ get request/response headers, headers are returned in a OrderedDict that allows you to get value using insensitive keys.""" self._check_headers_complete() return self.parser.get_headers() def should_keep_alive(self): """ return True if the connection should be kept alive """ self._check_headers_complete() return self.parser.should_keep_alive() def is_chunked(self): """ return True if Transfer-Encoding header value is chunked""" self._check_headers_complete() return self.parser.is_chunked() def wsgi_environ(self, initial=None): """ get WSGI environ based on the current request. :attr initial: dict, initial values to fill in environ. """ self._check_headers_complete() return self.parser.get_wsgi_environ() def body_file(self, buffering=None, binary=True, encoding=None, errors=None, newline=None): """ return the body as a buffered stream object. If binary is true an io.BufferedReader will be returned, else an io.TextIOWrapper. """ self._check_headers_complete() if buffering is None: buffering = -1 if buffering < 0: buffering = DEFAULT_BUFFER_SIZE raw = HttpBodyReader(self) buffer = BufferedReader(raw, buffering) if binary: return buffer text = TextIOWrapper(buffer, encoding, errors, newline) return text def body_string(self, binary=True, encoding=None, errors=None, newline=None): """ return body as string """ return self.body_file(binary=binary, encoding=encoding, newline=newline).read() def __iter__(self): return self def next(self): if self.parser.is_message_complete(): raise StopIteration # fetch data b = bytearray(DEFAULT_BUFFER_SIZE) recved = self.stream.readinto(b) if recved is None: raise NoMoreData("no more data") del b[recved:] # parse data nparsed = self.parser.execute(bytes(b), recved) if nparsed != recved and not self.parser.is_message_complete(): raise ParserError("nparsed != recved") if recved == 0: raise StopIteration return bytes(b)
def __call__(self, addr): '''Since an instance of HttpServer is passed to the Service class (with appropriate request_handler established during initialization), this __call__ method is what's actually invoked by diesel. ''' data = None while True: try: h = HttpParser() body = [] while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break data = receive() env = h.get_wsgi_environ() if 'HTTP_CONTENT_LENGTH' in env: env['CONTENT_LENGTH'] = env.pop("HTTP_CONTENT_LENGTH") if 'HTTP_CONTENT_TYPE' in env: env['CONTENT_TYPE'] = env.pop("HTTP_CONTENT_TYPE") env.update({ 'wsgi.version' : (1,0), 'wsgi.url_scheme' : 'http', # XXX incomplete 'wsgi.input' : cStringIO.StringIO(''.join(body)), 'wsgi.errors' : FileLikeErrorLogger(hlog), 'wsgi.multithread' : False, 'wsgi.multiprocess' : False, 'wsgi.run_once' : False, 'REMOTE_ADDR' : addr[0], 'SERVER_NAME' : HOSTNAME, 'SERVER_PORT': str(self.port), }) req = Request(env) if req.headers.get('Connection', '').lower() == 'upgrade': req.data = data resp = self.request_handler(req) if 'Server' not in resp.headers: resp.headers.add('Server', SERVER_TAG) if 'Date' not in resp.headers: resp.headers.add('Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) assert resp, "HTTP request handler _must_ return a response" self.send_response(resp, version=h.get_version()) if (not h.should_keep_alive()) or \ resp.headers.get('Connection', '').lower() == "close" or \ resp.headers.get('Content-Length') == None: return # Switching Protocols if resp.status_code == 101 and hasattr(resp, 'new_protocol'): resp.new_protocol(req) break except ConnectionClosed: break
def __call__(self, addr): """Since an instance of HttpServer is passed to the Service class (with appropriate request_handler established during initialization), this __call__ method is what's actually invoked by diesel. """ data = None while True: try: h = HttpParser() body = [] while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break data = receive() env = h.get_wsgi_environ() if "HTTP_CONTENT_LENGTH" in env: env["CONTENT_LENGTH"] = env.pop("HTTP_CONTENT_LENGTH") if "HTTP_CONTENT_TYPE" in env: env["CONTENT_TYPE"] = env.pop("HTTP_CONTENT_TYPE") env.update( { "wsgi.version": (1, 0), "wsgi.url_scheme": "http", # XXX incomplete "wsgi.input": cStringIO.StringIO("".join(body)), "wsgi.errors": FileLikeErrorLogger(hlog), "wsgi.multithread": False, "wsgi.multiprocess": False, "wsgi.run_once": False, "REMOTE_ADDR": addr[0], "SERVER_NAME": HOSTNAME, "SERVER_PORT": str(self.port), } ) req = Request(env) if req.headers.get("Connection", "").lower() == "upgrade": req.data = data resp = self.request_handler(req) if "Server" not in resp.headers: resp.headers.add("Server", SERVER_TAG) if "Date" not in resp.headers: resp.headers.add("Date", utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) assert resp, "HTTP request handler _must_ return a response" self.send_response(resp, version=h.get_version()) if ( (not h.should_keep_alive()) or resp.headers.get("Connection", "").lower() == "close" or resp.headers.get("Content-Length") == None ): return except ConnectionClosed: break
def _request_upstream_helper(self, method, uri, version, headers, body): ''' Forward the request to the upstream server. ''' log.debug('_request_upstream(): method={}, uri={}' \ .format(method, uri)) if self._mitm_host: parsed = urlparse(uri) url = 'https://{}{}'.format(self._mitm_host, parsed.path) else: url = uri token, remaining = self._token_store.dispense(url) log.debug('Signing request with {} token: {}.' .format(token.site, token.public)) if 'instagram' in url: qp = parse_qs(parsed.query) qp['access_token'] = token.public qp['sig'] = self._instagram.oauth_sign( method=method, url=url, token=token, query_params=qp, body_params=parse_qs(body) ) params = ['{}={}'.format(quote(k.encode('utf8')), quote(v.encode('utf8'))) for k,v in qp.items()] uri = '{}?{}'.format(parsed.path, '&'.join(params)) log.debug('Signed instagram URL: {}'.format(uri)) elif 'twitter' in url: headers['Authorization'] = self._twitter.oauth_sign( method=method, url=url, token=token.public, token_secret=token.secret, query_params=parse_qs(parsed.query), body_params=parse_qs(body) ) else: raise ValueError('No signing algorithm known for URL: {}' .format(url)) if self._mitm is None: url = urlparse(uri) host = url.hostname port = url.port if port is None: port = 80 if url.scheme == 'http' else 443 log.debug('Connecting to upstream (plaintext).') upstream = yield from asyncio.open_connection(host, port) upstream_reader, upstream_writer = upstream request = render_http_request(method, uri, version, headers, body) upstream_writer.write(request) response = b'' parser = HttpParser() while True: if not parser.is_headers_complete(): data = yield from upstream_reader.readline() else: data = yield from upstream_reader.read( int(parser.get_headers()['Content-Length']) ) log.debug('Received plaintext from upstream: {}'.format(data)) parser.execute(data, len(data)) if parser.is_partial_body(): body += parser.recv_body() if parser.is_message_complete(): version = self._parser.get_version() status = self._parser.get_status_code() reason = None # For some reason, the parser doesn't expose this :( headers = self._parser.get_headers() if status == 200: self._token_store.update_rate_limit(url, headers) log.debug('Plaintext upstream status: {}'.format(status)) log.debug('Plaintext upstream headers: {}'.format(headers)) log.debug('Plaintext upstream body: {}...'.format(body[:1000])) response = render_http_response( version, status, reason, headers, body ) break upstream_writer.close() else: upstream_write = self._mitm.forward request = render_http_request(method, uri, version, headers, body) upstream_write(request) response = yield from self._mitm.receive() version, status, reason, headers, body = response if status == 200: self._token_store.update_rate_limit(token, url, headers) response = render_http_response( version, status, reason, headers, body ) # Forward the upstream response to the client. self._transport.write(response) self._transport.close()
class ConnectionHandler: def __init__(self, connection, address, timeout): self.body_file ="" self.p = HttpParser() self.body = [] self.request_url = "" self.response_header = [] self.header_done = False self.url ="" self.controller = [] self.controller_ip = [] self.client = connection self.client_buffer = '' self.timeout = timeout self.method, self.path, self.protocol = self.get_base_header() if self.method=='CONNECT': self.method_CONNECT() elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE'): self.method_others() self.client.close() self.target.close() #clear #print self.controller , self.controller_ip def get_base_header(self): while 1: self.client_buffer += self.client.recv(BUFLEN) end = self.client_buffer.find('\n') if end!=-1: break #We dont wann those google.com urls. if not "127.0.0.1" in self.client_buffer[:end]: #Insert Url into database here self.url = '%s'%self.client_buffer[:end] data = (self.client_buffer[:end+1]).split() self.client_buffer = self.client_buffer[end+1:] #print data return data def method_CONNECT(self): self._connect_target(self.path) self.client.send(HTTPVER+' 200 Connection established\n'+ 'Proxy-agent: %s\n\n'%VERSION) self.client_buffer = '' self._read_write() def method_others(self): self.path = self.path[7:] i = self.path.find('/') host = self.path[:i] path = self.path[i:] self._connect_target(host) self.target.send('%s %s %s\n'%(self.method, path, self.protocol)+ self.client_buffer) self.client_buffer = '' self._read_write() def _connect_target(self, host): i = host.find(':') if i!=-1: port = int(host[i+1:]) host = host[:i] #print host else: port = 80 try: (soc_family, _, _, _, address) = socket.getaddrinfo(host, port)[0] print "Adress is ",address self.target = socket.socket(soc_family) self.target.connect(address) except Exception as e: print "Error Connecting to:"+str(host)+":"+str(port) self.request_url = "Error Connecting to:"+str(host)+":"+str(port) # insert to db here #Concat data to string self.request_url = str(host)+" | "+str(address)+" | "+str(self.url) #debug #print self.request_url def _read_write(self): time_out_max = self.timeout/3 socs = [self.client, self.target] count = 0 while 1: count += 1 (recv, _, error) = select.select(socs, [], socs, 3) if error: break if recv: for in_ in recv: try: #print " Receving Data " data = in_.recv(84333) except Exception as e: print e pass if in_ is self.client: out = self.target else: out = self.client try: if data: #column 25 #Dig here to analysis the traffic #print data try: #Lets parse the data using http_parser modules recved = len(data) # #print "We received so far "+ str(recved) nparsed = self.p.execute(data, recved) assert nparsed == recved # Check if self.p.is_headers_complete() and not self.header_done: #Header is an ordered dictionary header_s = self.p.get_headers() # Lets beautify it and print it. for header, value in header_s.items(): #Print Response # Connection : close format res_header = header+": "+value self.response_header.append(res_header) self.header_done = True #Put header to Database. #Check if the boday is partial, if then append the body if self.p.is_partial_body(): self.body.append(self.p.recv_body()) #print "appending body" +self.p.recv_body() #Append data body recived to a list #print self.body # If the parsing of current request is compleated if self.p.is_message_complete(): try: try: content_length = self.p.get_headers()['content-length'] except Exception as e: print "Exception in Body retrive-sub"+str(e) content_length = 0 pass self.body_file = "".join(self.body) body_file_type = ms.buffer(self.body_file[:1024]) #print self.request_url #print self.response_header #print body_file_type print urlid update_traffic(urlid,self.request_url,self.response_header,body_file_type) except Exception as e: print "Exception in Body retrive"+str(e) content_length = 0 pass except Exception as e: print e pass #if filetype in traffice == jar,class , pdf,flash, execute #save those files out.send(data) count = 0 except Exception as e: print e pass if count == time_out_max: break
class QHttpConnection(QObject): newRequest = pyqtSignal(QHttpRequest, QHttpResponse) disconnected = pyqtSignal() def __init__(self, sock, parent = None): super(QHttpConnection, self).__init__(parent) self.m_sock = sock self.m_body = [] self.m_parser = HttpParser() self.m_request = QHttpRequest(self) self.m_request = None self.m_response = QHttpResponse(self) self.m_response = None self.m_sock.readyRead.connect(self._onReadyRead) self.m_sock.disconnected.connect(self._onDisconnected) self.m_sock.bytesWritten.connect(self._onBytesWritten) return def write(self, data): self.m_sock.write(data) return def _onReadyRead(self): #qDebug('hehe') qtdata = self.m_sock.readAll() pydata = qtdata.data() np = self.m_parser.execute(pydata, len(pydata)) qDebug(str(np) + '=?' + str(len(pydata))) #qDebug(qtdata) #qDebug(qtdata.toHex()) #print(self.m_parser._body) #print(self.m_parser._body) #qDebug(str(self.m_parser.is_message_begin())) #qDebug(str(self.m_parser.is_message_complete())) #qDebug(str(self.m_parser.is_headers_complete())) if self.m_parser.is_headers_complete(): if self.m_request != None: qWarning('alread have a request object') else: self.m_request = QHttpRequest(self) _qogc.add(self.m_request) # qDebug(str(self.m_request)) # print(self.m_parser.get_headers()) True ### body area # qDebug(str(self.m_parser.is_message_begin())) # not use lines,这个可能指的是在客户端时,数据下载完成标识吧。 if self.m_parser.is_message_begin() and self.m_request != None: qDebug('body coming...') self.m_request.hasBody() mth = self.m_parser.get_method() # qDebug(mth) if mth == 'GET': if self.m_parser.is_headers_complete(): self.m_response = QHttpResponse(self) self.m_response.done.connect(self._onResponseDone) _qogc.add(self.m_response) self.newRequest.emit(self.m_request, self.m_response) pass elif mth == 'POST': if self.m_parser.is_partial_body(): self.m_body.append(self.m_parser.recv_body()) if self.m_parser.is_message_complete(): print(b''.join(self.m_body)) elif mth == 'CONNECT': if self.m_parser.is_headers_complete(): if self.m_response != None: qWarning('alread have a response object') else: self.m_response = QHttpResponse(self) self.m_response.done.connect(self._onResponseDone) _qogc.add(self.m_response) self.newRequest.emit(self.m_request, self.m_response) else: qDebug('hdr not complete') True else: qWarning("not impled method:" + mth) self.m_sock.close() return def _onDisconnected(self): # qDebug('hehe') self.disconnected.emit() return def _onBytesWritten(self, count): # qDebug('hehe') return def _onResponseDone(self): self.m_sock.disconnectFromHost() self.m_sock.close() # qDebug(str(self.m_request)) return def close(self): self.m_sock.flush() self.m_sock.close() return def last(self): return
class ConnectionHandler: def __init__(self, connection, address, timeout): self.body_file = "" self.p = HttpParser() self.body = [] self.request_url = "" self.response_header = [] self.header_done = False self.url = "" self.controller = [] self.controller_ip = [] self.client = connection self.client_buffer = '' self.timeout = timeout self.method, self.path, self.protocol = self.get_base_header() if self.method == 'CONNECT': self.method_CONNECT() elif self.method in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE'): self.method_others() self.client.close() self.target.close() #clear #print self.controller , self.controller_ip def get_base_header(self): while 1: self.client_buffer += self.client.recv(BUFLEN) end = self.client_buffer.find('\n') if end != -1: break #We dont wann those google.com urls. if not "127.0.0.1" in self.client_buffer[:end]: #Insert Url into database here self.url = '%s' % self.client_buffer[:end] data = (self.client_buffer[:end + 1]).split() self.client_buffer = self.client_buffer[end + 1:] #print data return data def method_CONNECT(self): self._connect_target(self.path) self.client.send(HTTPVER + ' 200 Connection established\n' + 'Proxy-agent: %s\n\n' % VERSION) self.client_buffer = '' self._read_write() def method_others(self): self.path = self.path[7:] i = self.path.find('/') host = self.path[:i] path = self.path[i:] self._connect_target(host) self.target.send('%s %s %s\n' % (self.method, path, self.protocol) + self.client_buffer) self.client_buffer = '' self._read_write() def _connect_target(self, host): i = host.find(':') if i != -1: port = int(host[i + 1:]) host = host[:i] #print host else: port = 80 try: (soc_family, _, _, _, address) = socket.getaddrinfo(host, port)[0] print "Adress is ", address self.target = socket.socket(soc_family) self.target.connect(address) except Exception as e: print "Error Connecting to:" + str(host) + ":" + str(port) self.request_url = "Error Connecting to:" + str(host) + ":" + str( port) # insert to db here #Concat data to string self.request_url = str(host) + " | " + str(address) + " | " + str( self.url) #debug #print self.request_url def _read_write(self): time_out_max = self.timeout / 3 socs = [self.client, self.target] count = 0 while 1: count += 1 (recv, _, error) = select.select(socs, [], socs, 3) if error: break if recv: for in_ in recv: try: #print " Receving Data " data = in_.recv(84333) except Exception as e: print e pass if in_ is self.client: out = self.target else: out = self.client try: if data: #column 25 #Dig here to analysis the traffic #print data try: #Lets parse the data using http_parser modules recved = len(data) # #print "We received so far "+ str(recved) nparsed = self.p.execute(data, recved) assert nparsed == recved # Check if self.p.is_headers_complete( ) and not self.header_done: #Header is an ordered dictionary header_s = self.p.get_headers() # Lets beautify it and print it. for header, value in header_s.items(): #Print Response # Connection : close format res_header = header + ": " + value self.response_header.append(res_header) self.header_done = True #Put header to Database. #Check if the boday is partial, if then append the body if self.p.is_partial_body(): self.body.append(self.p.recv_body()) #print "appending body" +self.p.recv_body() #Append data body recived to a list #print self.body # If the parsing of current request is compleated if self.p.is_message_complete(): try: try: content_length = self.p.get_headers( )['content-length'] except Exception as e: print "Exception in Body retrive-sub" + str( e) content_length = 0 pass self.body_file = "".join(self.body) body_file_type = ms.buffer( self.body_file[:1024]) #print self.request_url #print self.response_header #print body_file_type print urlid update_traffic(urlid, self.request_url, self.response_header, body_file_type) except Exception as e: print "Exception in Body retrive" + str( e) content_length = 0 pass except Exception as e: print e pass #if filetype in traffice == jar,class , pdf,flash, execute #save those files out.send(data) count = 0 except Exception as e: print e pass if count == time_out_max: break
class HttpStream(object): """ An HTTP parser providing higher-level access to a readable, sequential io.RawIOBase object. You can use implementions of http_parser.reader (IterReader, StringReader, SocketReader) or create your own. """ def __init__(self, stream, kind=HTTP_BOTH, decompress=False): """ constructor of HttpStream. :attr stream: an io.RawIOBase object :attr kind: Int, could be 0 to parseonly requests, 1 to parse only responses or 2 if we want to let the parser detect the type. """ self.parser = HttpParser(kind=kind, decompress=decompress) self.stream = stream def _check_headers_complete(self): if self.parser.is_headers_complete(): return while True: try: next(self) except StopIteration: if self.parser.is_headers_complete(): return raise NoMoreData("Can't parse headers") if self.parser.is_headers_complete(): return def _wait_status_line(self, cond): if self.parser.is_headers_complete(): return True data = [] if not cond(): while True: try: d = next(self) data.append(d) except StopIteration: if self.parser.is_headers_complete(): return True raise BadStatusLine(b"".join(data)) if cond(): return True return True def _wait_on_url(self): return self._wait_status_line(self.parser.get_url) def _wait_on_status(self): return self._wait_status_line(self.parser.get_status_code) def url(self): """ get full url of the request """ self._wait_on_url() return self.parser.get_url() def path(self): """ get path of the request (url without query string and fragment """ self._wait_on_url() return self.parser.get_path() def query_string(self): """ get query string of the url """ self._wait_on_url() return self.parser.get_query_string() def fragment(self): """ get fragment of the url """ self._wait_on_url() return self.parser.get_fragment() def version(self): self._wait_on_status() return self.parser.get_version() def status_code(self): """ get status code of a response as integer """ self._wait_on_status() return self.parser.get_status_code() def status(self): """ return complete status with reason """ status_code = self.status_code() reason = status_reasons.get(int(status_code), 'unknown') return "%s %s" % (status_code, reason) def method(self): """ get HTTP method as string""" self._wait_on_status() return self.parser.get_method() def headers(self): """ get request/response headers, headers are returned in a OrderedDict that allows you to get value using insensitive keys.""" self._check_headers_complete() return self.parser.get_headers() def should_keep_alive(self): """ return True if the connection should be kept alive """ self._check_headers_complete() return self.parser.should_keep_alive() def is_chunked(self): """ return True if Transfer-Encoding header value is chunked""" self._check_headers_complete() return self.parser.is_chunked() def wsgi_environ(self, initial=None): """ get WSGI environ based on the current request. :attr initial: dict, initial values to fill in environ. """ self._check_headers_complete() return self.parser.get_wsgi_environ() def body_file(self, buffering=None, binary=True, encoding=None, errors=None, newline=None): """ return the body as a buffered stream object. If binary is true an io.BufferedReader will be returned, else an io.TextIOWrapper. """ self._check_headers_complete() if buffering is None: buffering = -1 if buffering < 0: buffering = DEFAULT_BUFFER_SIZE raw = HttpBodyReader(self) buf = BufferedReader(raw, buffering) if binary: return buf text = TextIOWrapper(buf, encoding, errors, newline) return text def body_string(self, binary=True, encoding=None, errors=None, newline=None): """ return body as string """ return self.body_file(binary=binary, encoding=encoding, newline=newline).read() def __iter__(self): return self def __next__(self): if self.parser.is_message_complete(): raise StopIteration # fetch data b = bytearray(DEFAULT_BUFFER_SIZE) recved = self.stream.readinto(b) if recved is None: raise NoMoreData("no more data") del b[recved:] to_parse = bytes(b) # parse data nparsed = self.parser.execute(to_parse, recved) if nparsed != recved and not self.parser.is_message_complete(): raise ParserError("nparsed != recved (%s != %s) [%s]" % (nparsed, recved, bytes_to_str(to_parse))) if recved == 0: raise StopIteration return to_parse next = __next__
def request(self, method, url, headers=None, body=None, timeout=None): """Issues a `method` request to `path` on the connected server. Sends along `headers`, and body. Very low level--you must set "host" yourself, for example. It will set Content-Length, however. """ headers = headers or {} url_info = urlparse(url) fake_wsgi = dict((cgi_name(n), str(v).strip()) for n, v in headers.iteritems()) if body and "CONTENT_LENGTH" not in fake_wsgi: # If the caller hasn't set their own Content-Length but submitted # a body, we auto-set the Content-Length header here. fake_wsgi["CONTENT_LENGTH"] = str(len(body)) fake_wsgi.update( { "REQUEST_METHOD": method, "SCRIPT_NAME": "", "PATH_INFO": url_info[2], "QUERY_STRING": url_info[4], "wsgi.version": (1, 0), "wsgi.url_scheme": "http", # XXX incomplete "wsgi.input": cStringIO.StringIO(body or ""), "wsgi.errors": FileLikeErrorLogger(hlog), "wsgi.multithread": False, "wsgi.multiprocess": False, "wsgi.run_once": False, } ) req = Request(fake_wsgi) timeout_handler = TimeoutHandler(timeout or 60) url = str(req.path) if req.query_string: url += "?" + str(req.query_string) send("%s %s HTTP/1.1\r\n%s" % (req.method, url, str(req.headers))) if body: send(body) h = HttpParser() body = [] data = None while True: if data: used = h.execute(data, len(data)) if h.is_headers_complete(): body.append(h.recv_body()) if h.is_message_complete(): data = data[used:] break ev, val = first(receive_any=True, sleep=timeout_handler.remaining()) if ev == "sleep": timeout_handler.timeout() data = val resp = Response(response="".join(body), status=h.get_status_code(), headers=h.get_headers()) return resp
def run(self): self._log.info('Watch %s for new data.' % self.extension) while True: try: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.authmethod == "basic": self.client = ssl.wrap_socket(self.sock) elif self.authmethod == "cert": self.client = ssl.wrap_socket( self.sock, keyfile=self.config['client_cert_key_path'], certfile=self.config['client_cert_path']) else: raise KeyError('No authentication mechanisms defined') self._log.debug('Connecting to %s %i' % (self.host, self.port)) # self.client.settimeout(10) self.client.connect((self.host, self.port)) except socket.error as exc: self._log.exception('unable to connect to %s: %s' % (self.host, exc)) raise except KeyError: raise KeyError('No authentication mechanisms defined') if self.authhead is not None: self.client.send("GET %s HTTP/1.1\r\nHost: %s\r\n%s\r\n\r\n" % (self.extension, self.host, self.authhead)) else: self.client.send("GET %s HTTP/1.1\r\nHost: %s\r\n\r\n" % (self.extension, self.host)) readers = [self.client] writers = out_of_band = [] pending = b'' parser = HttpParser() self._log.debug("+") while not parser.is_headers_complete(): self._log.debug(".") try: chunk = self.client.recv(io.DEFAULT_BUFFER_SIZE) except socket.error as exc: err = exc.args[0] self._log.debug('a recv err (%s): %s' % (err, exc)) break if not chunk: self._log.exception('a No response from %s' % self.extension) break self._log.debug('a chunk %s' % chunk) nreceived = len(chunk) nparsed = parser.execute(chunk, nreceived) if nparsed != nreceived: self._log.exception('a nparsed %i != nreceived %i' % (nparsed, nreceived)) break self._log.debug('parser headers complete %s' % parser.get_headers()) while True: self._log.debug("-") try: readable, _, _ = select.select(readers, writers, out_of_band) except select.error as exc: self._log.debug("b select error: %s" % exc) if not readable: self._log.debug('b not readable') break try: chunk = self.client.recv(io.DEFAULT_BUFFER_SIZE) except socket.error as exc: err = exc.args[0] self._log.debug('b recv err (%s): %s' % (err, exc)) break if not chunk: self._log.debug('b not chunk') self.client.close() # pylint: disable=no-member break nreceived = len(chunk) self._log.debug('b chunk %s' % chunk) self._log.debug("repr: %s" % repr(chunk)) if re.match(r'0\r\n\r\n', chunk, re.M): self._log.debug('b end end end') break nparsed = parser.execute(chunk, nreceived) if nparsed != nreceived: self._log.exception('b nparsed %i != nreceived %i' % (nparsed, nreceived)) break data = pending + parser.recv_body() msg = "DATA: %s" % data self._log.debug(msg) lines = data.split(b'\n') pending = lines.pop(-1) for line in lines: trigger_payload = self._get_trigger_payload_from_line(line) if trigger_payload == 0: pass else: self._log.info('Triggering Dispatch Now') self._sensor_service.dispatch(trigger=self.TRIGGER_REF, payload=trigger_payload) self._log.debug('main loop done') self.client.close() # pylint: disable=no-member
class PupyHTTPWrapperServer(BasePupyTransport): path = '/index.php?d=' allowed_methods = ('GET') server = None headers = { 'Content-Type': 'text/html; charset=utf-8', 'Server': 'Apache', 'Connection': 'close', } __slots__ = ( 'parser', 'is_http', 'body', 'downstream_buffer', 'well_known', 'omit', 'probe_len' ) def __init__(self, *args, **kwargs): super(PupyHTTPWrapperServer, self).__init__(*args, **kwargs) self.parser = HttpParser() self.is_http = None self.body = [] self.downstream_buffer = Buffer() self.well_known = ('GET', 'POST', 'OPTIONS', 'HEAD', 'PUT', 'DELETE') self.omit = tuple( '{} {}'.format(x, y) for x in self.well_known for y in ( self.path, '/ws/', 'ws/')) self.probe_len = max(len(x) for x in self.omit) def _http_response(self, code, status, headers=None, datasize=None, content=None): headers = {} headers.update(self.headers) if headers: headers.update(headers) if datasize: headers.update({ 'Content-Length': datasize, 'Content-Type': 'application/octet-steram', }) data = '\r\n'.join([ 'HTTP/1.1 {} {}'.format(code, status), '\r\n'.join([ '{}: {}'.format(key, value) for key,value in headers.iteritems() ]) ]) + '\r\n\r\n' self.downstream.write(data) def _handle_file(self, filepath): try: with open(filepath) as infile: size = stat(filepath).st_size self._http_response(200, 'OK', datasize=size) while True: data = infile.read(65535) if data: self.downstream.write(data) else: break except: self._http_response(404, 'Not found', 'Not found') def _handle_not_found(self): self._http_response(404, 'Not found', 'Not found') def _handle_http(self, data): self.parser.execute(data, len(data)) if self.parser.is_headers_complete(): try: if not self.parser.get_method() in self.allowed_methods: self._http_response(405, 'Method Not Allowed') else: urlpath = self.parser.get_path() urlpath = [ x.strip() for x in urlpath.split('/') if ( x and not str(x) in ('.', '..') ) ] root = self.server.config.get_folder('wwwroot') secret = self.server.config.getboolean('httpd', 'secret') log = self.server.config.getboolean('httpd', 'log') if secret: wwwsecret = self.server.config.get('randoms', 'wwwsecret', random=5) if not (urlpath and urlpath[0] == wwwsecret): self._handle_not_found() if log: self.server.handler.display_error('{}: GET {} | SECRET = {}'.format( '{}:{}'.format(*self.downstream.transport.peer[:2]), urlpath, wwwsecret)) return urlpath = urlpath[1:] urlpath = path.sep.join([ self.server.config.get('randoms', x, new=False) or x for x in urlpath ]) if not urlpath: urlpath = 'index.html' filepath = path.join(root, urlpath) if path.exists(filepath): self._handle_file(filepath) if log: self.server.handler.display_success('{}: GET {}'.format( '{}:{}'.format(*self.downstream.transport.peer[:2]), urlpath)) else: self._handle_not_found() if log: self.server.handler.display_error('{}: GET {}'.format( '{}:{}'.format(*self.downstream.transport.peer[:2]), urlpath)) except Exception, e: print "Exception: {}".format(e) finally:
class HTTPSession(base_object.BaseObject): _http_header = "" _method = "" _version = "" _req_obj = "" _user_agent = "User-Agent: COS-598C-Project-Client\r\n" _accept = "Accept: */*\r\n" _accept_enc = "Accept-Encoding: *\r\n" _accept_charset = "Accept-Charset: *\r\n" _host = "" _writer = "" _closeable = False _http_parser = "" _nr_bytes = 0 def __init__(self, method, req_obj, version): self._method = method self._req_obj = req_obj self._version = version self._http_parser = HttpParser() def _build_first_line(self): first_line = self._method + " " + self._req_obj + " " + self._version + "\r\n" return first_line def set_host(self, host): self._host = "Host: " + host + "\r\n" def set_writer(self, writer): self._writer = writer def write_response(self, data): recved = len(data) nparsed = self._http_parser.execute(data, recved) assert nparsed == recved self._nr_bytes += recved if self._http_parser.is_partial_body(): self._writer.write(str(self._http_parser.recv_body())) if self._http_parser.is_message_complete(): self._closeable = True return self._nr_bytes def get_response_headers(self): if self._http_parser.is_headers_complete(): return self._http_parser.get_headers() def closeable(self): return self._closeable def set_port(self, port): return def get_request(self): self._http_header = self._build_first_line()+\ self._host+\ self._user_agent+\ self._accept+\ self._accept_enc+\ self._accept_charset+\ "\r\n" return self._http_header
class HTTPSession(base_object.BaseObject): _http_header = "" _method = "" _version = "" _req_obj = "" _user_agent = "User-Agent: COS-598C-Project-Client\r\n" _accept = "Accept: */*\r\n" _accept_enc = "Accept-Encoding: *\r\n" _accept_charset = "Accept-Charset: *\r\n" _host = "" _writer = "" _closeable = False _http_parser = "" _nr_bytes = 0 def __init__ (self, method, req_obj, version): self._method = method self._req_obj = req_obj self._version = version self._http_parser = HttpParser() def _build_first_line(self): first_line = self._method+" "+self._req_obj+" "+self._version+"\r\n" return first_line def set_host(self, host): self._host = "Host: "+host+"\r\n" def set_writer(self, writer): self._writer = writer def write_response(self, data): recved = len(data) nparsed = self._http_parser.execute(data, recved) assert nparsed == recved self._nr_bytes += recved if self._http_parser.is_partial_body(): self._writer.write(str(self._http_parser.recv_body())) if self._http_parser.is_message_complete(): self._closeable = True return self._nr_bytes def get_response_headers(self): if self._http_parser.is_headers_complete(): return self._http_parser.get_headers() def closeable(self): return self._closeable def set_port(self, port): return def get_request(self): self._http_header = self._build_first_line()+\ self._host+\ self._user_agent+\ self._accept+\ self._accept_enc+\ self._accept_charset+\ "\r\n" return self._http_header