def create_reader(bytes):
b = Buffer(1024)
for byte in bytes:
b.write_byte(byte)
b.flip()
p = PacketReader(b)
p.packet.position = b.position
p.packet.limit = b.limit
return p
def __init__(self):
self.state = self.STATE_INIT
self.buffer = Buffer(1024 * 16)
self.socket = None
self.reader = None
self.writer = None
self._time_command = False #whether to keep timing stats on a cmd
self._command_time = -1
self._incommand = False
self.current_resultset = None
def testParserSpeed(self):
N = 500000
b = Buffer(1024)
b.write_bytes("GET /bla.html?piet=blaat&jaap=aap HTTP/1.1\r\n")
b.write_bytes("Content-length: 10\r\n")
b.write_bytes("\r\n")
b.flip()
with unittest.timer() as tmr:
for i in range(N):
p = _http.HTTPParser(b)
p.parse()
b.position = 0
print tmr.sec(N)
def __init__(self):
self.state = self.STATE_INIT
self.buffer = Buffer(1024 * 16)
self.socket = None
self.reader = None
self.writer = None
self._time_command = False #whether to keep timing stats on a cmd
self._command_time = -1
self._incommand = False
self.current_resultset = None
def __init__(self, stream):
buffer_size = stream._read_buffer_size
if stream._reader is None:
if stream._reader_pool.get(buffer_size, []):
reader = stream._reader_pool[buffer_size].pop()
else:
reader = BufferedReader(None, Buffer(buffer_size))
else:
reader = stream._reader
reader.stream = stream._stream
self._reader = reader
self._stream = stream
def __init__(self, stream):
buffer_size = stream._write_buffer_size
if stream._writer is None:
if stream._writer_pool.get(buffer_size, []):
writer = stream._writer_pool[buffer_size].pop()
else:
writer = BufferedWriter(None, Buffer(buffer_size))
else:
writer = stream._writer
writer.stream = stream._stream
self._writer = writer
self._stream = stream
def handle(client_socket):
buffer = Buffer(1024)
client_socket.read(buffer) #read the request
buffer.clear()
buffer.write_bytes(
"HTTP/1.0 200 OK\r\n" \
"Content-Length: 13\r\n" \
"Connection: close\r\n" \
"\r\n" \
"Hello World!\n"
)
buffer.flip()
client_socket.write(buffer)
client_socket.close()
def whois(self, query, hostname, flags):
"""Perform initial lookup with TLD whois server
then, if the quick flag is false, search that result
for the region-specifc whois server and do a lookup
there for contact details
"""
response = ''
engine = QueryEngine()
result = engine.asynchronous(hostname, adns.rr.ADDR)
if len(result[3]):
ips = [rr[1] for rr in result[3]]
ip = random.choice(ips)
try:
s = Socket.connect((ip, 43))
writer = BufferedWriter(s, Buffer(1024))
reader = BufferedReader(s, Buffer(1024))
if (hostname == NICClient.GERMNICHOST):
writer.write_bytes("-T dn,ace -C US-ASCII " + query + "\r\n")
else:
writer.write_bytes(query + "\r\n")
writer.flush()
while True:
try:
d = reader.read_bytes_available()
except EOFError:
break
response += d
if not d:
break
except IOError:
pass
nhost = None
if (flags & NICClient.WHOIS_RECURSE and nhost == None):
nhost = self.findwhois_server(response, hostname)
if (nhost != None):
response += self.whois(query, nhost, 0)
return response
def create_reader(bytes):
b = Buffer(1024)
for byte in bytes:
b.write_byte(byte)
b.flip()
p = PacketReader(b)
p.packet.position = b.position
p.packet.limit = b.limit
return p
class Connection(object):
"""Represents a single connection to a MySQL Database host."""
STATE_ERROR = -1
STATE_INIT = 0
STATE_CONNECTING = 1
STATE_CONNECTED = 2
STATE_CLOSING = 3
STATE_CLOSED = 4
def __init__(self):
self.state = self.STATE_INIT
self.buffer = Buffer(1024 * 16)
self.socket = None
self.reader = None
self.writer = None
self._time_command = False #whether to keep timing stats on a cmd
self._command_time = -1
self._incommand = False
self.current_resultset = None
def _scramble(self, password, seed):
"""taken from java jdbc driver, scrambles the password using the given seed
according to the mysql login protocol"""
stage1 = SHA(password).digest()
stage2 = SHA(stage1).digest()
md = SHA()
md.update(seed)
md.update(stage2)
#i love python :-):
return ''.join(map(chr, [x ^ ord(stage1[i]) for i, x in enumerate(map(ord, md.digest()))]))
def _handshake(self, user, password, database):
"""performs the mysql login handshake"""
#init buffer for reading (both pos and lim = 0)
self.buffer.clear()
self.buffer.flip()
#read server welcome
packet = self.reader.read_packet()
self.protocol_version = packet.read_byte() #normally this would be 10 (0xa)
if self.protocol_version == 0xff:
#error on initial greeting, possibly too many connection error
raise ClientLoginError.from_error_packet(packet, skip = 2)
elif self.protocol_version == 0xa:
pass #expected
else:
assert False, "Unexpected protocol version %02x" % self.protocol_version
self.server_version = packet.read_bytes_until(0)
packet.skip(4) #thread_id
scramble_buff = packet.read_bytes(8)
packet.skip(1) #filler
server_caps = packet.read_short()
#CAPS.dbg(server_caps)
if not server_caps & CAPS.PROTOCOL_41:
assert False, "<4.1 auth not supported"
server_language = packet.read_byte()
server_status = packet.read_short()
packet.skip(13) #filler
if packet.remaining:
scramble_buff += packet.read_bytes_until(0)
else:
assert False, "<4.1 auth not supported"
client_caps = CAPS.LONG_FLAG | CAPS.PROTOCOL_41 | CAPS.SECURE_CONNECTION | CAPS.TRANSACTIONS
if database:
if not server_caps & CAPS.CONNECT_WITH_DB:
assert False, "initial db given but not supported by server"
else:
#tell the server that we will supply initial database
client_caps |= CAPS.CONNECT_WITH_DB
#build and write our answer to the initial handshake packet
self.writer.clear()
self.writer.start()
self.writer.write_int(client_caps)
self.writer.write_int(1024 * 1024 * 32) #16mb max packet
self.writer.write_byte(server_language)
self.writer.write_bytes('\0' * 23) #filler
self.writer.write_bytes(user + '\0')
if password:
self.writer.write_byte(20)
self.writer.write_bytes(self._scramble(password, scramble_buff))
else:
self.writer.write_byte(0)
if database:
self.writer.write_bytes(database + '\0')
self.writer.finish(1)
self.writer.flush()
#read final answer from server
self.buffer.flip()
packet = self.reader.read_packet()
result = packet.read_byte()
if result == 0xff:
raise ClientLoginError.from_error_packet(packet)
elif result == 0xfe:
assert False, "old password handshake not implemented"
def _close_current_resultset(self, resultset):
assert resultset == self.current_resultset
self.current_resultset = None
def _send_command(self, cmd, cmd_text):
"""sends a command with the given text"""
#self.log.debug('cmd %s %s', cmd, cmd_text)
#note: we are not using normal writer.start/finish here, because the cmd
#could not fit in buffer, causing flushes in write_string, in that case 'finish' would
#not be able to go back to the header of the packet to write the length in that case
self.writer.clear()
self.writer.write_header(len(cmd_text) + 1 + 4, 0) #1 is len of cmd, 4 is len of header, 0 is packet number
self.writer.write_byte(cmd)
self.writer.write_bytes(cmd_text)
self.writer.flush()
def _close(self):
#self.log.debug("close mysql client %s", id(self))
try:
self.state = self.STATE_CLOSING
if self.current_resultset:
self.current_resultset.close(True)
self.socket.close()
self.state = self.STATE_CLOSED
except:
self.state = self.STATE_ERROR
raise
def connect(self, host = "localhost", port = 3306, user = "", passwd = "", db = "", autocommit = None, charset = None):
"""connects to the given host and port with user and passwd"""
#self.log.debug("connect mysql client %s %s %s %s %s", id(self), host, port, user, passwd)
try:
#print 'connect', host, user, passwd, db
#parse addresses of form str <host:port>
if type(host) == str:
if host[0] == '/': #assume unix domain socket
addr = host
elif ':' in host:
host, port = host.split(':')
port = int(port)
addr = (host, port)
else:
addr = (host, port)
assert self.state == self.STATE_INIT, "make sure connection is not already connected or closed"
self.state = self.STATE_CONNECTING
self.socket = Socket.connect(addr, TIMEOUT_CURRENT)
self.reader = BufferedPacketReader(self.socket, self.buffer)
self.writer = BufferedPacketWriter(self.socket, self.buffer)
self._handshake(user, passwd, db)
#handshake complete client can now send commands
self.state = self.STATE_CONNECTED
if autocommit == False:
self.set_autocommit(False)
elif autocommit == True:
self.set_autocommit(True)
else:
pass #whatever is the default of the db (ON in the case of mysql)
if charset is not None:
self.set_charset(charset)
return self
except TimeoutError:
self.state = self.STATE_INIT
raise
except ClientLoginError:
self.state = self.STATE_INIT
raise
except:
self.state = self.STATE_ERROR
raise
def close(self):
"""close this connection"""
assert self.is_connected(), "make sure connection is connected before closing"
if self._incommand != False: assert False, "cannot close while still in a command"
self._close()
def command(self, cmd, cmd_text):
"""sends a COM_XXX command with the given text and possibly return a resultset (select)"""
#print 'command', cmd, repr(cmd_text), type(cmd_text)
assert type(cmd_text) == str #as opposed to unicode
assert self.is_connected(), "make sure connection is connected before query"
if self._incommand != False: assert False, "overlapped commands not supported"
if self.current_resultset: assert False, "overlapped commands not supported, pls read prev resultset and close it"
try:
self._incommand = True
if self._time_command:
start_time = time.time()
self._send_command(cmd, cmd_text)
#read result, expect 1 of OK, ERROR or result set header
self.buffer.flip()
packet = self.reader.read_packet()
result = packet.read_byte()
#print 'res', result
if self._time_command:
end_time = time.time()
self._command_time = end_time - start_time
if result == 0x00:
#OK, return (affected rows, last row id)
rowcount = self.reader.read_length_coded_binary()
lastrowid = self.reader.read_length_coded_binary()
return (rowcount, lastrowid)
elif result == 0xff:
raise ClientCommandError.from_error_packet(packet)
else: #result set
self.current_resultset = ResultSet(self, result)
return self.current_resultset
finally:
self._incommand = False
def is_connected(self):
return self.state == self.STATE_CONNECTED
def query(self, cmd_text):
"""Sends a COM_QUERY command with the given text and return a resultset (select)"""
return self.command(COMMAND.QUERY, cmd_text)
def init_db(self, cmd_text):
"""Sends a COM_INIT command with the given text"""
return self.command(COMMAND.INITDB, cmd_text)
def set_autocommit(self, commit):
"""Sets autocommit setting for this connection. True = on, False = off"""
self.command(COMMAND.QUERY, "SET AUTOCOMMIT = %s" % ('1' if commit else '0'))
def commit(self):
"""Commits this connection"""
self.command(COMMAND.QUERY, "COMMIT")
def rollback(self):
"""Issues a rollback on this connection"""
self.command(COMMAND.QUERY, "ROLLBACK")
def set_charset(self, charset):
"""Sets the charset for this connections (used to decode string fields into unicode strings)"""
self.reader.reader.encoding = charset
def set_time_command(self, time_command):
self._time_command = time_command
def get_command_time(self):
return self._command_time
def writer(self):
if self._writer is None:
self._writer = BufferedWriter(self._stream, Buffer(self._write_buffer_size))
return self._writer
def reader(self):
if self._reader is None:
self._reader = BufferedReader(self._stream, Buffer(self._read_buffer_size))
return self._reader
def testParser(self):
b = Buffer(1024)
b.write_bytes("POST /bla.html?piet=blaat&jaap=aap#blaataap HTTP/1.1\r\n")
b.write_bytes("Content-length: 10\r\n")
b.write_bytes("Another-header: blaat: piet\r\n")
b.write_bytes("Another-header: blaat: piet\r\n")
b.write_bytes("Content-type: blaat/piet\r\n")
b.write_bytes("\r\n")
b.write_bytes("1234567890")
b.flip()
print b.limit, b.position
p = _http.HTTPParser(b)
print repr(p.environ)
print 'isfin', repr(p.is_finished())
print p.parse()
print 'isfin', repr(p.is_finished())
print b.limit, b.position
print repr(p.environ)
print p.parse()
fd = s.fileno()
#we are connected, turn to non-blockig mode
s.setblocking(0)
#print s.fileno()
STATE_WRITE_SET_REQUEST = 0
STATE_READ_SET_RESPONSE = 1
STATE_WAIT_SET_RESPONSE = 2
state = STATE_WRITE_SET_REQUEST
N = 100000
#N = 1000
n = 0
buff = Buffer(1024)
def callback(flags):
#print 'cb', flags
if flags & event.EV_TIMEOUT:
raise Exception("Timeout")
global state
global n
while True:
if state == STATE_WRITE_SET_REQUEST:
buff.clear()
key = 'piet%d' % n
value = 'klaas%d' % n
buff.write_bytes("%s %s %d 0 %d\r\n%s\r\n" %
('set', key, 0, len(value), value))
def fetch10(self, s, uri):
b = Buffer(1024)
b.clear()
b.write_bytes("GET %s HTTP/1.0\r\n" % uri)
b.write_bytes("\r\n")
b.flip()
s.write(b)
b.clear()
s.read(b)
b.flip()
return b.read_bytes(b.remaining)
class Connection(object):
"""Represents a single connection to a MySQL Database host."""
STATE_ERROR = -1
STATE_INIT = 0
STATE_CONNECTING = 1
STATE_CONNECTED = 2
STATE_CLOSING = 3
STATE_CLOSED = 4
def __init__(self):
self.state = self.STATE_INIT
self.buffer = Buffer(1024 * 16)
self.socket = None
self.reader = None
self.writer = None
self._time_command = False #whether to keep timing stats on a cmd
self._command_time = -1
self._incommand = False
self.current_resultset = None
def _scramble(self, password, seed):
"""taken from java jdbc driver, scrambles the password using the given seed
according to the mysql login protocol"""
stage1 = SHA(password).digest()
stage2 = SHA(stage1).digest()
md = SHA()
md.update(seed)
md.update(stage2)
#i love python :-):
return ''.join(
map(chr, [
x ^ ord(stage1[i]) for i, x in enumerate(map(ord, md.digest()))
]))
def _handshake(self, user, password, database):
"""performs the mysql login handshake"""
#init buffer for reading (both pos and lim = 0)
self.buffer.clear()
self.buffer.flip()
#read server welcome
packet = self.reader.read_packet()
self.protocol_version = packet.read_byte(
) #normally this would be 10 (0xa)
if self.protocol_version == 0xff:
#error on initial greeting, possibly too many connection error
raise ClientLoginError.from_error_packet(packet, skip=2)
elif self.protocol_version == 0xa:
pass #expected
else:
assert False, "Unexpected protocol version %02x" % self.protocol_version
self.server_version = packet.read_bytes_until(0)
packet.skip(4) #thread_id
scramble_buff = packet.read_bytes(8)
packet.skip(1) #filler
server_caps = packet.read_short()
#CAPS.dbg(server_caps)
if not server_caps & CAPS.PROTOCOL_41:
assert False, "<4.1 auth not supported"
server_language = packet.read_byte()
server_status = packet.read_short()
packet.skip(13) #filler
if packet.remaining:
scramble_buff += packet.read_bytes_until(0)
else:
assert False, "<4.1 auth not supported"
client_caps = server_caps
#always turn off compression
client_caps &= ~CAPS.COMPRESS
if not server_caps & CAPS.CONNECT_WITH_DB and database:
assert False, "initial db given but not supported by server"
if server_caps & CAPS.CONNECT_WITH_DB and not database:
client_caps &= ~CAPS.CONNECT_WITH_DB
#build and write our answer to the initial handshake packet
self.writer.clear()
self.writer.start()
self.writer.write_int(client_caps)
self.writer.write_int(1024 * 1024 * 32) #16mb max packet
self.writer.write_byte(server_language)
self.writer.write_bytes('\0' * 23) #filler
self.writer.write_bytes(user + '\0')
if password:
self.writer.write_byte(20)
self.writer.write_bytes(self._scramble(password, scramble_buff))
else:
self.writer.write_byte(0)
if database:
self.writer.write_bytes(database + '\0')
self.writer.finish(1)
self.writer.flush()
#read final answer from server
self.buffer.flip()
packet = self.reader.read_packet()
result = packet.read_byte()
if result == 0xff:
raise ClientLoginError.from_error_packet(packet)
elif result == 0xfe:
assert False, "old password handshake not implemented"
def _close_current_resultset(self, resultset):
assert resultset == self.current_resultset
self.current_resultset = None
def _send_command(self, cmd, cmd_text):
"""sends a command with the given text"""
#self.log.debug('cmd %s %s', cmd, cmd_text)
#note: we are not using normal writer.start/finish here, because the cmd
#could not fit in buffer, causing flushes in write_string, in that case 'finish' would
#not be able to go back to the header of the packet to write the length in that case
self.writer.clear()
self.writer.write_header(
len(cmd_text) + 1 + 4,
0) #1 is len of cmd, 4 is len of header, 0 is packet number
self.writer.write_byte(cmd)
self.writer.write_bytes(cmd_text)
self.writer.flush()
def _close(self):
#self.log.debug("close mysql client %s", id(self))
try:
self.state = self.STATE_CLOSING
if self.current_resultset:
self.current_resultset.close(True)
self.socket.close()
self.state = self.STATE_CLOSED
except:
self.state = self.STATE_ERROR
raise
def connect(self,
host="localhost",
port=3306,
user="",
passwd="",
db="",
autocommit=None,
charset=None):
"""connects to the given host and port with user and passwd"""
#self.log.debug("connect mysql client %s %s %s %s %s", id(self), host, port, user, passwd)
try:
#print 'connect', host, user, passwd, db
#parse addresses of form str <host:port>
if type(host) == str:
if host[0] == '/': #assume unix domain socket
addr = host
elif ':' in host:
host, port = host.split(':')
port = int(port)
addr = (host, port)
else:
addr = (host, port)
assert self.state == self.STATE_INIT, "make sure connection is not already connected or closed"
self.state = self.STATE_CONNECTING
self.socket = Socket.connect(addr, timeout=Timeout.current())
self.reader = BufferedPacketReader(self.socket, self.buffer)
self.writer = BufferedPacketWriter(self.socket, self.buffer)
self._handshake(user, passwd, db)
#handshake complete client can now send commands
self.state = self.STATE_CONNECTED
if autocommit == False:
self.set_autocommit(False)
elif autocommit == True:
self.set_autocommit(True)
else:
pass #whatever is the default of the db (ON in the case of mysql)
if charset is not None:
self.set_charset(charset)
return self
except TimeoutError:
self.state = self.STATE_INIT
raise
except ClientLoginError:
self.state = self.STATE_INIT
raise
except:
self.state = self.STATE_ERROR
raise
def close(self):
"""close this connection"""
assert self.is_connected(
), "make sure connection is connected before closing"
if self._incommand != False:
assert False, "cannot close while still in a command"
self._close()
def command(self, cmd, cmd_text):
"""sends a COM_XXX command with the given text and possibly return a resultset (select)"""
#print 'command', cmd, repr(cmd_text), type(cmd_text)
assert type(cmd_text) == str #as opposed to unicode
assert self.is_connected(
), "make sure connection is connected before query"
if self._incommand != False:
assert False, "overlapped commands not supported"
if self.current_resultset:
assert False, "overlapped commands not supported, pls read prev resultset and close it"
try:
self._incommand = True
if self._time_command:
start_time = time.time()
self._send_command(cmd, cmd_text)
#read result, expect 1 of OK, ERROR or result set header
self.buffer.flip()
packet = self.reader.read_packet()
result = packet.read_byte()
#print 'res', result
if self._time_command:
end_time = time.time()
self._command_time = end_time - start_time
if result == 0x00:
#OK, return (affected rows, last row id)
rowcount = self.reader.read_length_coded_binary()
lastrowid = self.reader.read_length_coded_binary()
return (rowcount, lastrowid)
elif result == 0xff:
raise ClientCommandError.from_error_packet(packet)
else: #result set
self.current_resultset = ResultSet(self, result)
return self.current_resultset
finally:
self._incommand = False
def is_connected(self):
return self.state == self.STATE_CONNECTED
def query(self, cmd_text):
"""Sends a COM_QUERY command with the given text and return a resultset (select)"""
return self.command(COMMAND.QUERY, cmd_text)
def init_db(self, cmd_text):
"""Sends a COM_INIT command with the given text"""
return self.command(COMMAND.INITDB, cmd_text)
def set_autocommit(self, commit):
"""Sets autocommit setting for this connection. True = on, False = off"""
self.command(COMMAND.QUERY,
"SET AUTOCOMMIT = %s" % ('1' if commit else '0'))
def commit(self):
"""Commits this connection"""
self.command(COMMAND.QUERY, "COMMIT")
def rollback(self):
"""Issues a rollback on this connection"""
self.command(COMMAND.QUERY, "ROLLBACK")
def set_charset(self, charset):
"""Sets the charset for this connections (used to decode string fields into unicode strings)"""
self.reader.reader.encoding = charset
def set_time_command(self, time_command):
self._time_command = time_command
def get_command_time(self):
return self._command_time
def testParserSpeed(self):
N = 500000
b = Buffer(1024)
b.write_bytes("GET /bla.html?piet=blaat&jaap=aap HTTP/1.1\r\n")
b.write_bytes("Content-length: 10\r\n")
b.write_bytes("\r\n")
b.flip()
with unittest.timer() as tmr:
for i in range(N):
p = _http.HTTPParser(b)
p.parse()
b.position = 0
print tmr.sec(N)
def testParser(self):
b = Buffer(1024)
b.write_bytes(
"POST /bla.html?piet=blaat&jaap=aap#blaataap HTTP/1.1\r\n")
b.write_bytes("Content-length: 10\r\n")
b.write_bytes("Another-header: blaat: piet\r\n")
b.write_bytes("Another-header: blaat: piet\r\n")
b.write_bytes("Content-type: blaat/piet\r\n")
b.write_bytes("\r\n")
b.write_bytes("1234567890")
b.flip()
print b.limit, b.position
p = _http.HTTPParser(b)
print repr(p.environ)
print 'isfin', repr(p.is_finished())
print p.parse()
print 'isfin', repr(p.is_finished())
print b.limit, b.position
print repr(p.environ)
print p.parse()
def fetch10(self, s, uri):
b = Buffer(1024)
b.clear()
b.write_bytes("GET %s HTTP/1.0\r\n" % uri)
b.write_bytes("\r\n")
b.flip()
s.write(b)
b.clear()
s.read(b)
b.flip()
return b.read_bytes(b.remaining)
def __init__(self, stream, buffer_size = 1024 * 8, read_buffer_size = 0, write_buffer_size = 0):
self.stream = stream
self.reader = BufferedReader(stream, Buffer(read_buffer_size or buffer_size))
self.writer = BufferedWriter(stream, Buffer(write_buffer_size or buffer_size))
