async def _read_bytes(self, num_bytes):
try:
data = await self._reader.readexactly(num_bytes)
except asyncio.streams.IncompleteReadError as e:
msg = "Lost connection to MySQL server during query"
raise OperationalError(2013, msg) from e
except (IOError, OSError) as e:
msg = "Lost connection to MySQL server during query (%s)" % (e, )
raise OperationalError(2013, msg) from e
return data
def _write_bytes(self, data):
try:
return self._writer.write(data)
except RuntimeError as e:
self.close()
msg = "Lost connection to MySQL server during query (%s)" % (e, )
raise OperationalError(2013, msg) from e
async def _process_auth(self, plugin_name, auth_packet):
if plugin_name == b"mysql_native_password":
# https://dev.mysql.com/doc/internals/en/
# secure-password-authentication.html#packet-Authentication::
# Native41
data = _scramble(self._password.encode('latin1'),
auth_packet.read_all())
elif plugin_name == b"mysql_old_password":
# https://dev.mysql.com/doc/internals/en/
# old-password-authentication.html
data = _scramble_323(self._password.encode('latin1'),
auth_packet.read_all()) + b'\0'
elif plugin_name == b"mysql_clear_password":
# https://dev.mysql.com/doc/internals/en/
# clear-text-authentication.html
data = self._password.encode('latin1') + b'\0'
else:
raise OperationalError(
2059, "Authentication plugin '%s' not configured" % plugin_name
)
self.write_packet(data)
pkt = await self._read_packet()
pkt.check_error()
self._auth_plugin_used = plugin_name
return pkt
def _read_packet(self, packet_type=MysqlPacket):
"""Read an entire "mysql packet" in its entirety from the network
and return a MysqlPacket type that represents the results.
"""
buff = b''
try:
while True:
packet_header = yield from self._reader.readexactly(4)
btrl, btrh, packet_number = struct.unpack(
'<HBB', packet_header)
bytes_to_read = btrl + (btrh << 16)
# Outbound and inbound packets are numbered sequentialy, so
# we increment in both write_packet and read_packet. The count
# is reset at new COMMAND PHASE.
if packet_number != self._next_seq_id:
raise InternalError(
"Packet sequence number wrong - got %d expected %d" %
(packet_number, self._next_seq_id))
self._next_seq_id = (self._next_seq_id + 1) % 256
recv_data = yield from self._reader.readexactly(bytes_to_read)
buff += recv_data
if bytes_to_read < MAX_PACKET_LEN:
break
except (OSError, EOFError) as exc:
msg = "MySQL server has gone away (%s)"
raise OperationalError(2006, msg % (exc, )) from exc
packet = packet_type(buff, self._encoding)
packet.check_error()
return packet
def opener(filename):
try:
self._file_object = open(filename, 'rb')
except IOError:
raise OperationalError(
1017, "Can't find file"
" '{0}'".format(filename))
async def _read_ok_packet(self):
pkt = await self._read_packet()
if not pkt.is_ok_packet():
raise OperationalError(2014, "Command Out of Sync")
ok = OKPacketWrapper(pkt)
self.server_status = ok.server_status
return True
def post_auth_routine(self):
"""
Anything that was initialized in a PyMySQL connection
after a successful authentication
"""
try:
if self.sql_mode is not None:
c = self.cursor()
c.execute("SET sql_mode=%s", (self.sql_mode, ))
if self.init_command is not None:
c = self.cursor()
c.execute(self.init_command)
self.commit()
if self.autocommit_mode is not None:
self.autocommit(self.autocommit_mode)
except Exception as e:
self._rfile = None
if sock is not None:
try:
sock.close()
except socket.error:
pass
raise OperationalError(
2003,
"Can't connect to MySQL server on %r (%s)" % (self.host, e))
def _read_packet(self, packet_type=MysqlPacket):
"""Read an entire "mysql packet" in its entirety from the network
and return a MysqlPacket type that represents the results.
"""
buff = b''
try:
while True:
packet_header = yield from self._reader.readexactly(4)
# logger.debug(_convert_to_str(packet_header))
packet_length_bin = packet_header[:3]
# TODO: check sequence id
# packet_number
byte2int(packet_header[3])
# pad little-endian number
bin_length = packet_length_bin + b'\0'
bytes_to_read = struct.unpack('<I', bin_length)[0]
recv_data = yield from self._reader.readexactly(bytes_to_read)
# logger.debug(dump_packet(recv_data))
buff += recv_data
if bytes_to_read < MAX_PACKET_LEN:
break
except (OSError, EOFError) as exc:
msg = "MySQL server has gone away (%s)"
raise OperationalError(2006, msg % (exc, )) from exc
packet = packet_type(buff, self._encoding)
packet.check_error()
return packet
def test_error_on_file_read(cursor, table_local_file):
with patch.object(builtins, 'open') as open_mocked:
m = MagicMock()
m.read.side_effect = OperationalError(1024, 'Error reading file')
m.close.return_value = None
open_mocked.return_value = m
with pytest.raises(OperationalError):
yield from cursor.execute("LOAD DATA LOCAL INFILE 'some.txt'"
" INTO TABLE test_load_local fields "
"terminated by ','")
async def _connect(self):
# TODO: Set close callback
# raise OperationalError(2006,
# "MySQL server has gone away (%r)" % (e,))
try:
if self._unix_socket and self._host in ('localhost', '127.0.0.1'):
self._reader, self._writer = await \
asyncio.wait_for(
asyncio.open_unix_connection(
self._unix_socket,
loop=self._loop),
timeout=self.connect_timeout)
self.host_info = "Localhost via UNIX socket: " + \
self._unix_socket
else:
self._reader, self._writer = await \
asyncio.wait_for(
asyncio.open_connection(
self._host,
self._port,
loop=self._loop),
timeout=self.connect_timeout)
self._set_keep_alive()
self.host_info = "socket %s:%d" % (self._host, self._port)
# do not set no delay in case of unix_socket
if self._no_delay and not self._unix_socket:
self._set_nodelay(True)
self._next_seq_id = 0
await self._get_server_information()
await self._request_authentication()
self.connected_time = self._loop.time()
if self.sql_mode is not None:
await self.query("SET sql_mode=%s" % (self.sql_mode,))
if self.init_command is not None:
await self.query(self.init_command)
await self.commit()
if self.autocommit_mode is not None:
await self.autocommit(self.autocommit_mode)
except Exception as e:
if self._writer:
self._writer.transport.close()
self._reader = None
self._writer = None
raise OperationalError(2003,
"Can't connect to MySQL server on %r" %
self._host) from e
def _read_load_local_packet(self, first_packet):
load_packet = LoadLocalPacketWrapper(first_packet)
local_packet = LoadLocalFile(load_packet.filename, self.connection)
self.filename = load_packet.filename
yield from local_packet.send_data()
ok_packet = yield from self.connection._read_packet()
if not ok_packet.is_ok_packet():
raise OperationalError(2014, "Commands Out of Sync")
self._read_ok_packet(ok_packet)
if self.warning_count > 0:
yield from self._print_warnings()
self.filename = None
def freader(chunk_size):
try:
chunk = self._file_object.read(chunk_size)
if not chunk:
self._file_object.close()
self._file_object = None
except Exception as e:
self._file_object.close()
self._file_object = None
msg = "Error reading file {}".format(self.filename)
raise OperationalError(1024, msg) from e
return chunk
async def _read_load_local_packet(self, first_packet):
load_packet = LoadLocalPacketWrapper(first_packet)
sender = LoadLocalFile(load_packet.filename, self.connection)
try:
await sender.send_data()
except Exception:
# Skip ok packet
await self.connection._read_packet()
raise
ok_packet = await self.connection._read_packet()
if not ok_packet.is_ok_packet():
raise OperationalError(2014, "Commands Out of Sync")
self._read_ok_packet(ok_packet)
async def _process_auth(self, plugin_name, auth_packet):
# These auth plugins do their own packet handling
if plugin_name == b"caching_sha2_password":
await self.caching_sha2_password_auth(auth_packet)
self._auth_plugin_used = plugin_name.decode()
elif plugin_name == b"sha256_password":
await self.sha256_password_auth(auth_packet)
self._auth_plugin_used = plugin_name.decode()
else:
if plugin_name == b"mysql_native_password":
# https://dev.mysql.com/doc/internals/en/
# secure-password-authentication.html#packet-Authentication::
# Native41
data = _auth.scramble_native_password(
self._password.encode('latin1'),
auth_packet.read_all())
elif plugin_name == b"mysql_old_password":
# https://dev.mysql.com/doc/internals/en/
# old-password-authentication.html
data = _auth.scramble_old_password(
self._password.encode('latin1'),
auth_packet.read_all()
) + b'\0'
elif plugin_name == b"mysql_clear_password":
# https://dev.mysql.com/doc/internals/en/
# clear-text-authentication.html
data = self._password.encode('latin1') + b'\0'
else:
raise OperationalError(
2059, "Authentication plugin '{0}'"
" not configured".format(plugin_name)
)
self.write_packet(data)
pkt = await self._read_packet()
pkt.check_error()
self._auth_plugin_used = plugin_name.decode()
return pkt
async def sha256_password_auth(self, pkt):
if self._ssl_context:
logger.debug("sha256: Sending plain password")
data = self._password.encode('latin1') + b'\0'
self.write_packet(data)
pkt = await self._read_packet()
pkt.check_error()
return pkt
if pkt.is_auth_switch_request():
self.salt = pkt.read_all()
if not self.server_public_key and self._password:
# Request server public key
logger.debug("sha256: Requesting server public key")
self.write_packet(b'\1')
pkt = await self._read_packet()
pkt.check_error()
if pkt.is_extra_auth_data():
self.server_public_key = pkt._data[1:]
logger.debug(
"Received public key:\n",
self.server_public_key.decode('ascii')
)
if self._password:
if not self.server_public_key:
raise OperationalError("Couldn't receive server's public key")
data = _auth.sha2_rsa_encrypt(
self._password.encode('latin1'), self.salt,
self.server_public_key
)
else:
data = b''
self.write_packet(data)
pkt = await self._read_packet()
pkt.check_error()
return pkt
def _connect(self, **kwargs):
"""
Filthy shim to stop a full handshake from the actual
pymysql library so that we can intercept the connection
and grab the salt.
"""
sock = None
try:
if self.unix_socket and self.host in ('localhost', '127.0.0.1'):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
t = sock.gettimeout()
sock.settimeout(self.connect_timeout)
sock.connect(self.unix_socket)
sock.settimeout(t)
self.host_info = "Localhost via UNIX socket"
if DEBUG: print('connected using unix_socket')
else:
sock = socket.create_connection((self.host, self.port),
self.connect_timeout)
self.host_info = "socket %s:%d" % (self.host, self.port)
if DEBUG: print('connected using socket')
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
if self.no_delay:
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.socket = sock
self._rfile = _makefile(sock, 'rb')
self._get_server_information()
# we'll be doing this ourselves
#self._request_authentication()
except Exception as e:
self._rfile = None
if sock is not None:
try:
sock.close()
except socket.error:
pass
raise OperationalError(
2003,
"Can't connect to MySQL server on %r (%s)" % (self.host, e))
def connect(self):
# TODO: Set close callback
# raise OperationalError(2006,
# "MySQL server has gone away (%r)" % (e,))
try:
if self._unix_socket and self._host in ('localhost', '127.0.0.1'):
self._reader, self._writer = yield from \
asyncio.open_unix_connection(self._unix_socket,
loop=self._loop)
self.host_info = "Localhost via UNIX socket: " + \
self._unix_socket
else:
self._reader, self._writer = yield from \
asyncio.open_connection(self._host, self._port,
loop=self._loop)
self.host_info = "socket %s:%d" % (self._host, self._port)
if self._no_delay:
self._set_nodelay(True)
yield from self._get_server_information()
yield from self._request_authentication()
self.connected_time = self._loop.time()
if self.sql_mode is not None:
yield from self.query("SET sql_mode=%s" % (self.sql_mode, ))
if self.init_command is not None:
yield from self.query(self.init_command)
yield from self.commit()
if self.autocommit_mode is not None:
yield from self.autocommit(self.autocommit_mode)
except OSError as e:
self._reader, self._writer = None, None
raise OperationalError(
2003,
"Can't connect to MySQL server on %r (%s)" % (self._host, e))
def _read_packet(self, packet_type=MysqlPacket):
"""Read an entire "mysql packet" in its entirety from the network
and return a MysqlPacket type that represents the results.
"""
buff = b''
try:
while True:
packet_header = yield from self._reader.readexactly(4)
btrl, btrh, packet_number = struct.unpack(
'<HBB', packet_header)
bytes_to_read = btrl + (btrh << 16)
# TODO: check sequence id
recv_data = yield from self._reader.readexactly(bytes_to_read)
buff += recv_data
if bytes_to_read < MAX_PACKET_LEN:
break
except (OSError, EOFError) as exc:
msg = "MySQL server has gone away (%s)"
raise OperationalError(2006, msg % (exc, )) from exc
packet = packet_type(buff, self._encoding)
packet.check_error()
return packet
async def caching_sha2_password_auth(self, pkt):
# No password fast path
if not self._password:
self.write_packet(b'')
pkt = await self._read_packet()
pkt.check_error()
return pkt
if pkt.is_auth_switch_request():
# Try from fast auth
logger.debug("caching sha2: Trying fast path")
self.salt = pkt.read_all()
scrambled = _auth.scramble_caching_sha2(
self._password.encode('latin1'), self.salt)
self.write_packet(scrambled)
pkt = await self._read_packet()
pkt.check_error()
# else: fast auth is tried in initial handshake
if not pkt.is_extra_auth_data():
raise OperationalError("caching sha2: Unknown packet "
"for fast auth: {0}".format(pkt._data[:1]))
# magic numbers:
# 2 - request public key
# 3 - fast auth succeeded
# 4 - need full auth
pkt.advance(1)
n = pkt.read_uint8()
if n == 3:
logger.debug("caching sha2: succeeded by fast path.")
pkt = await self._read_packet()
pkt.check_error() # pkt must be OK packet
return pkt
if n != 4:
raise OperationalError("caching sha2: Unknown "
"result for fast auth: {0}".format(n))
logger.debug("caching sha2: Trying full auth...")
if self._ssl_context:
logger.debug("caching sha2: Sending plain "
"password via secure connection")
self.write_packet(self._password.encode('latin1') + b'\0')
pkt = await self._read_packet()
pkt.check_error()
return pkt
if not self.server_public_key:
self.write_packet(b'\x02')
pkt = await self._read_packet() # Request public key
pkt.check_error()
if not pkt.is_extra_auth_data():
raise OperationalError("caching sha2: Unknown packet "
"for public key: {0}".format(
pkt._data[:1]))
self.server_public_key = pkt._data[1:]
logger.debug(self.server_public_key.decode('ascii'))
data = _auth.sha2_rsa_encrypt(self._password.encode('latin1'),
self.salt, self.server_public_key)
self.write_packet(data)
pkt = await self._read_packet()
pkt.check_error()
async def _request_authentication(self):
# https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse
if self.server_version.split('.', 1)[0] == 'Inception2':
self.client_flag |= CLIENT.MULTI_RESULTS
elif int(self.server_version.split('.', 1)[0]) >= 5:
self.client_flag |= CLIENT.MULTI_RESULTS
if self.user is None:
raise ValueError("Did not specify a username")
if self._ssl_context:
# capablities, max packet, charset
data = struct.pack('<IIB', self.client_flag, 16777216, 33)
data += b'\x00' * (32 - len(data))
self.write_packet(data)
# Stop sending events to data_received
self._writer.transport.pause_reading()
# Get the raw socket from the transport
raw_sock = self._writer.transport.get_extra_info('socket',
default=None)
if raw_sock is None:
raise RuntimeError("Transport does not expose socket instance")
raw_sock = raw_sock.dup()
self._writer.transport.close()
# MySQL expects TLS negotiation to happen in the middle of a
# TCP connection not at start. Passing in a socket to
# open_connection will cause it to negotiate TLS on an existing
# connection not initiate a new one.
self._reader, self._writer = await asyncio.open_connection(
sock=raw_sock,
ssl=self._ssl_context,
loop=self._loop,
server_hostname=self._host)
charset_id = charset_by_name(self.charset).id
if isinstance(self.user, str):
_user = self.user.encode(self.encoding)
else:
_user = self.user
data_init = struct.pack('<iIB23s', self.client_flag, MAX_PACKET_LEN,
charset_id, b'')
data = data_init + _user + b'\0'
authresp = b''
auth_plugin = self._client_auth_plugin
if not self._client_auth_plugin:
# Contains the auth plugin from handshake
auth_plugin = self._server_auth_plugin
if auth_plugin in ('', 'mysql_native_password'):
authresp = _auth.scramble_native_password(
self._password.encode('latin1'), self.salt)
elif auth_plugin == 'caching_sha2_password':
if self._password:
authresp = _auth.scramble_caching_sha2(
self._password.encode('latin1'), self.salt)
# Else: empty password
elif auth_plugin == 'sha256_password':
if self._ssl_context and self.server_capabilities & CLIENT.SSL:
authresp = self._password.encode('latin1') + b'\0'
elif self._password:
authresp = b'\1' # request public key
else:
authresp = b'\0' # empty password
elif auth_plugin in ('', 'mysql_clear_password'):
authresp = self._password.encode('latin1') + b'\0'
if self.server_capabilities & CLIENT.PLUGIN_AUTH_LENENC_CLIENT_DATA:
data += lenenc_int(len(authresp)) + authresp
elif self.server_capabilities & CLIENT.SECURE_CONNECTION:
data += struct.pack('B', len(authresp)) + authresp
else: # pragma: no cover
# not testing against servers without secure auth (>=5.0)
data += authresp + b'\0'
if self._db and self.server_capabilities & CLIENT.CONNECT_WITH_DB:
if isinstance(self._db, str):
db = self._db.encode(self.encoding)
else:
db = self._db
data += db + b'\0'
if self.server_capabilities & CLIENT.PLUGIN_AUTH:
name = auth_plugin
if isinstance(name, str):
name = name.encode('ascii')
data += name + b'\0'
self._auth_plugin_used = auth_plugin
# Sends the server a few pieces of client info
if self.server_capabilities & CLIENT.CONNECT_ATTRS:
connect_attrs = b''
for k, v in self._connect_attrs.items():
k, v = k.encode('utf8'), v.encode('utf8')
connect_attrs += struct.pack('B', len(k)) + k
connect_attrs += struct.pack('B', len(v)) + v
data += struct.pack('B', len(connect_attrs)) + connect_attrs
self.write_packet(data)
auth_packet = await self._read_packet()
# if authentication method isn't accepted the first byte
# will have the octet 254
if auth_packet.is_auth_switch_request():
# https://dev.mysql.com/doc/internals/en/
# connection-phase-packets.html#packet-Protocol::AuthSwitchRequest
auth_packet.read_uint8() # 0xfe packet identifier
plugin_name = auth_packet.read_string()
if (self.server_capabilities & CLIENT.PLUGIN_AUTH
and plugin_name is not None):
await self._process_auth(plugin_name, auth_packet)
else:
# send legacy handshake
data = _auth.scramble_old_password(
self._password.encode('latin1'),
auth_packet.read_all()) + b'\0'
self.write_packet(data)
await self._read_packet()
elif auth_packet.is_extra_auth_data():
if auth_plugin == "caching_sha2_password":
await self.caching_sha2_password_auth(auth_packet)
elif auth_plugin == "sha256_password":
await self.sha256_password_auth(auth_packet)
else:
raise OperationalError(
"Received extra packet "
"for auth method %r", auth_plugin)
def test_sql_or_recoverable_error_sql(self, mock_rollback):
e = OperationalError(2013, 'meow')
assert retry.sql_or_recoverable_error(e)
mock_rollback.assert_called()
def opener(filename):
try:
self._file_object = open(filename, 'rb')
except IOError as e:
msg = "Can't find file '{0}'".format(filename)
raise OperationalError(1017, msg) from e
