def create_reader(bytes): b = Buffer(1024) for byte in bytes: b.write_byte(byte) b.flip() p = PacketReader(b) p.packet.position = b.position p.packet.limit = b.limit return p
def __init__(self): self.state = self.STATE_INIT self.buffer = Buffer(1024 * 16) self.socket = None self.reader = None self.writer = None self._time_command = False #whether to keep timing stats on a cmd self._command_time = -1 self._incommand = False self.current_resultset = None
def testParserSpeed(self): N = 500000 b = Buffer(1024) b.write_bytes("GET /bla.html?piet=blaat&jaap=aap HTTP/1.1\r\n") b.write_bytes("Content-length: 10\r\n") b.write_bytes("\r\n") b.flip() with unittest.timer() as tmr: for i in range(N): p = _http.HTTPParser(b) p.parse() b.position = 0 print tmr.sec(N)
def __init__(self, stream): buffer_size = stream._read_buffer_size if stream._reader is None: if stream._reader_pool.get(buffer_size, []): reader = stream._reader_pool[buffer_size].pop() else: reader = BufferedReader(None, Buffer(buffer_size)) else: reader = stream._reader reader.stream = stream._stream self._reader = reader self._stream = stream
def __init__(self, stream): buffer_size = stream._write_buffer_size if stream._writer is None: if stream._writer_pool.get(buffer_size, []): writer = stream._writer_pool[buffer_size].pop() else: writer = BufferedWriter(None, Buffer(buffer_size)) else: writer = stream._writer writer.stream = stream._stream self._writer = writer self._stream = stream
def handle(client_socket): buffer = Buffer(1024) client_socket.read(buffer) #read the request buffer.clear() buffer.write_bytes( "HTTP/1.0 200 OK\r\n" \ "Content-Length: 13\r\n" \ "Connection: close\r\n" \ "\r\n" \ "Hello World!\n" ) buffer.flip() client_socket.write(buffer) client_socket.close()
def whois(self, query, hostname, flags): """Perform initial lookup with TLD whois server then, if the quick flag is false, search that result for the region-specifc whois server and do a lookup there for contact details """ response = '' engine = QueryEngine() result = engine.asynchronous(hostname, adns.rr.ADDR) if len(result[3]): ips = [rr[1] for rr in result[3]] ip = random.choice(ips) try: s = Socket.connect((ip, 43)) writer = BufferedWriter(s, Buffer(1024)) reader = BufferedReader(s, Buffer(1024)) if (hostname == NICClient.GERMNICHOST): writer.write_bytes("-T dn,ace -C US-ASCII " + query + "\r\n") else: writer.write_bytes(query + "\r\n") writer.flush() while True: try: d = reader.read_bytes_available() except EOFError: break response += d if not d: break except IOError: pass nhost = None if (flags & NICClient.WHOIS_RECURSE and nhost == None): nhost = self.findwhois_server(response, hostname) if (nhost != None): response += self.whois(query, nhost, 0) return response
class Connection(object): """Represents a single connection to a MySQL Database host.""" STATE_ERROR = -1 STATE_INIT = 0 STATE_CONNECTING = 1 STATE_CONNECTED = 2 STATE_CLOSING = 3 STATE_CLOSED = 4 def __init__(self): self.state = self.STATE_INIT self.buffer = Buffer(1024 * 16) self.socket = None self.reader = None self.writer = None self._time_command = False #whether to keep timing stats on a cmd self._command_time = -1 self._incommand = False self.current_resultset = None def _scramble(self, password, seed): """taken from java jdbc driver, scrambles the password using the given seed according to the mysql login protocol""" stage1 = SHA(password).digest() stage2 = SHA(stage1).digest() md = SHA() md.update(seed) md.update(stage2) #i love python :-): return ''.join(map(chr, [x ^ ord(stage1[i]) for i, x in enumerate(map(ord, md.digest()))])) def _handshake(self, user, password, database): """performs the mysql login handshake""" #init buffer for reading (both pos and lim = 0) self.buffer.clear() self.buffer.flip() #read server welcome packet = self.reader.read_packet() self.protocol_version = packet.read_byte() #normally this would be 10 (0xa) if self.protocol_version == 0xff: #error on initial greeting, possibly too many connection error raise ClientLoginError.from_error_packet(packet, skip = 2) elif self.protocol_version == 0xa: pass #expected else: assert False, "Unexpected protocol version %02x" % self.protocol_version self.server_version = packet.read_bytes_until(0) packet.skip(4) #thread_id scramble_buff = packet.read_bytes(8) packet.skip(1) #filler server_caps = packet.read_short() #CAPS.dbg(server_caps) if not server_caps & CAPS.PROTOCOL_41: assert False, "<4.1 auth not supported" server_language = packet.read_byte() server_status = packet.read_short() packet.skip(13) #filler if packet.remaining: scramble_buff += packet.read_bytes_until(0) else: assert False, "<4.1 auth not supported" client_caps = CAPS.LONG_FLAG | CAPS.PROTOCOL_41 | CAPS.SECURE_CONNECTION | CAPS.TRANSACTIONS if database: if not server_caps & CAPS.CONNECT_WITH_DB: assert False, "initial db given but not supported by server" else: #tell the server that we will supply initial database client_caps |= CAPS.CONNECT_WITH_DB #build and write our answer to the initial handshake packet self.writer.clear() self.writer.start() self.writer.write_int(client_caps) self.writer.write_int(1024 * 1024 * 32) #16mb max packet self.writer.write_byte(server_language) self.writer.write_bytes('\0' * 23) #filler self.writer.write_bytes(user + '\0') if password: self.writer.write_byte(20) self.writer.write_bytes(self._scramble(password, scramble_buff)) else: self.writer.write_byte(0) if database: self.writer.write_bytes(database + '\0') self.writer.finish(1) self.writer.flush() #read final answer from server self.buffer.flip() packet = self.reader.read_packet() result = packet.read_byte() if result == 0xff: raise ClientLoginError.from_error_packet(packet) elif result == 0xfe: assert False, "old password handshake not implemented" def _close_current_resultset(self, resultset): assert resultset == self.current_resultset self.current_resultset = None def _send_command(self, cmd, cmd_text): """sends a command with the given text""" #self.log.debug('cmd %s %s', cmd, cmd_text) #note: we are not using normal writer.start/finish here, because the cmd #could not fit in buffer, causing flushes in write_string, in that case 'finish' would #not be able to go back to the header of the packet to write the length in that case self.writer.clear() self.writer.write_header(len(cmd_text) + 1 + 4, 0) #1 is len of cmd, 4 is len of header, 0 is packet number self.writer.write_byte(cmd) self.writer.write_bytes(cmd_text) self.writer.flush() def _close(self): #self.log.debug("close mysql client %s", id(self)) try: self.state = self.STATE_CLOSING if self.current_resultset: self.current_resultset.close(True) self.socket.close() self.state = self.STATE_CLOSED except: self.state = self.STATE_ERROR raise def connect(self, host = "localhost", port = 3306, user = "", passwd = "", db = "", autocommit = None, charset = None): """connects to the given host and port with user and passwd""" #self.log.debug("connect mysql client %s %s %s %s %s", id(self), host, port, user, passwd) try: #print 'connect', host, user, passwd, db #parse addresses of form str <host:port> if type(host) == str: if host[0] == '/': #assume unix domain socket addr = host elif ':' in host: host, port = host.split(':') port = int(port) addr = (host, port) else: addr = (host, port) assert self.state == self.STATE_INIT, "make sure connection is not already connected or closed" self.state = self.STATE_CONNECTING self.socket = Socket.connect(addr, TIMEOUT_CURRENT) self.reader = BufferedPacketReader(self.socket, self.buffer) self.writer = BufferedPacketWriter(self.socket, self.buffer) self._handshake(user, passwd, db) #handshake complete client can now send commands self.state = self.STATE_CONNECTED if autocommit == False: self.set_autocommit(False) elif autocommit == True: self.set_autocommit(True) else: pass #whatever is the default of the db (ON in the case of mysql) if charset is not None: self.set_charset(charset) return self except TimeoutError: self.state = self.STATE_INIT raise except ClientLoginError: self.state = self.STATE_INIT raise except: self.state = self.STATE_ERROR raise def close(self): """close this connection""" assert self.is_connected(), "make sure connection is connected before closing" if self._incommand != False: assert False, "cannot close while still in a command" self._close() def command(self, cmd, cmd_text): """sends a COM_XXX command with the given text and possibly return a resultset (select)""" #print 'command', cmd, repr(cmd_text), type(cmd_text) assert type(cmd_text) == str #as opposed to unicode assert self.is_connected(), "make sure connection is connected before query" if self._incommand != False: assert False, "overlapped commands not supported" if self.current_resultset: assert False, "overlapped commands not supported, pls read prev resultset and close it" try: self._incommand = True if self._time_command: start_time = time.time() self._send_command(cmd, cmd_text) #read result, expect 1 of OK, ERROR or result set header self.buffer.flip() packet = self.reader.read_packet() result = packet.read_byte() #print 'res', result if self._time_command: end_time = time.time() self._command_time = end_time - start_time if result == 0x00: #OK, return (affected rows, last row id) rowcount = self.reader.read_length_coded_binary() lastrowid = self.reader.read_length_coded_binary() return (rowcount, lastrowid) elif result == 0xff: raise ClientCommandError.from_error_packet(packet) else: #result set self.current_resultset = ResultSet(self, result) return self.current_resultset finally: self._incommand = False def is_connected(self): return self.state == self.STATE_CONNECTED def query(self, cmd_text): """Sends a COM_QUERY command with the given text and return a resultset (select)""" return self.command(COMMAND.QUERY, cmd_text) def init_db(self, cmd_text): """Sends a COM_INIT command with the given text""" return self.command(COMMAND.INITDB, cmd_text) def set_autocommit(self, commit): """Sets autocommit setting for this connection. True = on, False = off""" self.command(COMMAND.QUERY, "SET AUTOCOMMIT = %s" % ('1' if commit else '0')) def commit(self): """Commits this connection""" self.command(COMMAND.QUERY, "COMMIT") def rollback(self): """Issues a rollback on this connection""" self.command(COMMAND.QUERY, "ROLLBACK") def set_charset(self, charset): """Sets the charset for this connections (used to decode string fields into unicode strings)""" self.reader.reader.encoding = charset def set_time_command(self, time_command): self._time_command = time_command def get_command_time(self): return self._command_time
def writer(self): if self._writer is None: self._writer = BufferedWriter(self._stream, Buffer(self._write_buffer_size)) return self._writer
def reader(self): if self._reader is None: self._reader = BufferedReader(self._stream, Buffer(self._read_buffer_size)) return self._reader
def testParser(self): b = Buffer(1024) b.write_bytes("POST /bla.html?piet=blaat&jaap=aap#blaataap HTTP/1.1\r\n") b.write_bytes("Content-length: 10\r\n") b.write_bytes("Another-header: blaat: piet\r\n") b.write_bytes("Another-header: blaat: piet\r\n") b.write_bytes("Content-type: blaat/piet\r\n") b.write_bytes("\r\n") b.write_bytes("1234567890") b.flip() print b.limit, b.position p = _http.HTTPParser(b) print repr(p.environ) print 'isfin', repr(p.is_finished()) print p.parse() print 'isfin', repr(p.is_finished()) print b.limit, b.position print repr(p.environ) print p.parse()
fd = s.fileno() #we are connected, turn to non-blockig mode s.setblocking(0) #print s.fileno() STATE_WRITE_SET_REQUEST = 0 STATE_READ_SET_RESPONSE = 1 STATE_WAIT_SET_RESPONSE = 2 state = STATE_WRITE_SET_REQUEST N = 100000 #N = 1000 n = 0 buff = Buffer(1024) def callback(flags): #print 'cb', flags if flags & event.EV_TIMEOUT: raise Exception("Timeout") global state global n while True: if state == STATE_WRITE_SET_REQUEST: buff.clear() key = 'piet%d' % n value = 'klaas%d' % n buff.write_bytes("%s %s %d 0 %d\r\n%s\r\n" % ('set', key, 0, len(value), value))
def fetch10(self, s, uri): b = Buffer(1024) b.clear() b.write_bytes("GET %s HTTP/1.0\r\n" % uri) b.write_bytes("\r\n") b.flip() s.write(b) b.clear() s.read(b) b.flip() return b.read_bytes(b.remaining)
class Connection(object): """Represents a single connection to a MySQL Database host.""" STATE_ERROR = -1 STATE_INIT = 0 STATE_CONNECTING = 1 STATE_CONNECTED = 2 STATE_CLOSING = 3 STATE_CLOSED = 4 def __init__(self): self.state = self.STATE_INIT self.buffer = Buffer(1024 * 16) self.socket = None self.reader = None self.writer = None self._time_command = False #whether to keep timing stats on a cmd self._command_time = -1 self._incommand = False self.current_resultset = None def _scramble(self, password, seed): """taken from java jdbc driver, scrambles the password using the given seed according to the mysql login protocol""" stage1 = SHA(password).digest() stage2 = SHA(stage1).digest() md = SHA() md.update(seed) md.update(stage2) #i love python :-): return ''.join( map(chr, [ x ^ ord(stage1[i]) for i, x in enumerate(map(ord, md.digest())) ])) def _handshake(self, user, password, database): """performs the mysql login handshake""" #init buffer for reading (both pos and lim = 0) self.buffer.clear() self.buffer.flip() #read server welcome packet = self.reader.read_packet() self.protocol_version = packet.read_byte( ) #normally this would be 10 (0xa) if self.protocol_version == 0xff: #error on initial greeting, possibly too many connection error raise ClientLoginError.from_error_packet(packet, skip=2) elif self.protocol_version == 0xa: pass #expected else: assert False, "Unexpected protocol version %02x" % self.protocol_version self.server_version = packet.read_bytes_until(0) packet.skip(4) #thread_id scramble_buff = packet.read_bytes(8) packet.skip(1) #filler server_caps = packet.read_short() #CAPS.dbg(server_caps) if not server_caps & CAPS.PROTOCOL_41: assert False, "<4.1 auth not supported" server_language = packet.read_byte() server_status = packet.read_short() packet.skip(13) #filler if packet.remaining: scramble_buff += packet.read_bytes_until(0) else: assert False, "<4.1 auth not supported" client_caps = server_caps #always turn off compression client_caps &= ~CAPS.COMPRESS if not server_caps & CAPS.CONNECT_WITH_DB and database: assert False, "initial db given but not supported by server" if server_caps & CAPS.CONNECT_WITH_DB and not database: client_caps &= ~CAPS.CONNECT_WITH_DB #build and write our answer to the initial handshake packet self.writer.clear() self.writer.start() self.writer.write_int(client_caps) self.writer.write_int(1024 * 1024 * 32) #16mb max packet self.writer.write_byte(server_language) self.writer.write_bytes('\0' * 23) #filler self.writer.write_bytes(user + '\0') if password: self.writer.write_byte(20) self.writer.write_bytes(self._scramble(password, scramble_buff)) else: self.writer.write_byte(0) if database: self.writer.write_bytes(database + '\0') self.writer.finish(1) self.writer.flush() #read final answer from server self.buffer.flip() packet = self.reader.read_packet() result = packet.read_byte() if result == 0xff: raise ClientLoginError.from_error_packet(packet) elif result == 0xfe: assert False, "old password handshake not implemented" def _close_current_resultset(self, resultset): assert resultset == self.current_resultset self.current_resultset = None def _send_command(self, cmd, cmd_text): """sends a command with the given text""" #self.log.debug('cmd %s %s', cmd, cmd_text) #note: we are not using normal writer.start/finish here, because the cmd #could not fit in buffer, causing flushes in write_string, in that case 'finish' would #not be able to go back to the header of the packet to write the length in that case self.writer.clear() self.writer.write_header( len(cmd_text) + 1 + 4, 0) #1 is len of cmd, 4 is len of header, 0 is packet number self.writer.write_byte(cmd) self.writer.write_bytes(cmd_text) self.writer.flush() def _close(self): #self.log.debug("close mysql client %s", id(self)) try: self.state = self.STATE_CLOSING if self.current_resultset: self.current_resultset.close(True) self.socket.close() self.state = self.STATE_CLOSED except: self.state = self.STATE_ERROR raise def connect(self, host="localhost", port=3306, user="", passwd="", db="", autocommit=None, charset=None): """connects to the given host and port with user and passwd""" #self.log.debug("connect mysql client %s %s %s %s %s", id(self), host, port, user, passwd) try: #print 'connect', host, user, passwd, db #parse addresses of form str <host:port> if type(host) == str: if host[0] == '/': #assume unix domain socket addr = host elif ':' in host: host, port = host.split(':') port = int(port) addr = (host, port) else: addr = (host, port) assert self.state == self.STATE_INIT, "make sure connection is not already connected or closed" self.state = self.STATE_CONNECTING self.socket = Socket.connect(addr, timeout=Timeout.current()) self.reader = BufferedPacketReader(self.socket, self.buffer) self.writer = BufferedPacketWriter(self.socket, self.buffer) self._handshake(user, passwd, db) #handshake complete client can now send commands self.state = self.STATE_CONNECTED if autocommit == False: self.set_autocommit(False) elif autocommit == True: self.set_autocommit(True) else: pass #whatever is the default of the db (ON in the case of mysql) if charset is not None: self.set_charset(charset) return self except TimeoutError: self.state = self.STATE_INIT raise except ClientLoginError: self.state = self.STATE_INIT raise except: self.state = self.STATE_ERROR raise def close(self): """close this connection""" assert self.is_connected( ), "make sure connection is connected before closing" if self._incommand != False: assert False, "cannot close while still in a command" self._close() def command(self, cmd, cmd_text): """sends a COM_XXX command with the given text and possibly return a resultset (select)""" #print 'command', cmd, repr(cmd_text), type(cmd_text) assert type(cmd_text) == str #as opposed to unicode assert self.is_connected( ), "make sure connection is connected before query" if self._incommand != False: assert False, "overlapped commands not supported" if self.current_resultset: assert False, "overlapped commands not supported, pls read prev resultset and close it" try: self._incommand = True if self._time_command: start_time = time.time() self._send_command(cmd, cmd_text) #read result, expect 1 of OK, ERROR or result set header self.buffer.flip() packet = self.reader.read_packet() result = packet.read_byte() #print 'res', result if self._time_command: end_time = time.time() self._command_time = end_time - start_time if result == 0x00: #OK, return (affected rows, last row id) rowcount = self.reader.read_length_coded_binary() lastrowid = self.reader.read_length_coded_binary() return (rowcount, lastrowid) elif result == 0xff: raise ClientCommandError.from_error_packet(packet) else: #result set self.current_resultset = ResultSet(self, result) return self.current_resultset finally: self._incommand = False def is_connected(self): return self.state == self.STATE_CONNECTED def query(self, cmd_text): """Sends a COM_QUERY command with the given text and return a resultset (select)""" return self.command(COMMAND.QUERY, cmd_text) def init_db(self, cmd_text): """Sends a COM_INIT command with the given text""" return self.command(COMMAND.INITDB, cmd_text) def set_autocommit(self, commit): """Sets autocommit setting for this connection. True = on, False = off""" self.command(COMMAND.QUERY, "SET AUTOCOMMIT = %s" % ('1' if commit else '0')) def commit(self): """Commits this connection""" self.command(COMMAND.QUERY, "COMMIT") def rollback(self): """Issues a rollback on this connection""" self.command(COMMAND.QUERY, "ROLLBACK") def set_charset(self, charset): """Sets the charset for this connections (used to decode string fields into unicode strings)""" self.reader.reader.encoding = charset def set_time_command(self, time_command): self._time_command = time_command def get_command_time(self): return self._command_time
def testParser(self): b = Buffer(1024) b.write_bytes( "POST /bla.html?piet=blaat&jaap=aap#blaataap HTTP/1.1\r\n") b.write_bytes("Content-length: 10\r\n") b.write_bytes("Another-header: blaat: piet\r\n") b.write_bytes("Another-header: blaat: piet\r\n") b.write_bytes("Content-type: blaat/piet\r\n") b.write_bytes("\r\n") b.write_bytes("1234567890") b.flip() print b.limit, b.position p = _http.HTTPParser(b) print repr(p.environ) print 'isfin', repr(p.is_finished()) print p.parse() print 'isfin', repr(p.is_finished()) print b.limit, b.position print repr(p.environ) print p.parse()
def __init__(self, stream, buffer_size = 1024 * 8, read_buffer_size = 0, write_buffer_size = 0): self.stream = stream self.reader = BufferedReader(stream, Buffer(read_buffer_size or buffer_size)) self.writer = BufferedWriter(stream, Buffer(write_buffer_size or buffer_size))