def get_group_server(self, group, mode=None, status=None):
"""Get a MySQL server from a group
The method uses MySQL Fabric to get the correct MySQL server
for the specified group. You can specify mode or status, but
not both.
The mode argument will decide whether the primary or a secondary
server is returned. When no secondary server is available, the
primary is returned.
Status is used to force getting either a primary or a secondary.
The returned tuple contains host, port and uuid.
Raises InterfaceError on errors; ValueError when both mode
and status are given.
Returns a FabricMySQLServer object.
"""
if mode and status:
raise ValueError("Either mode or status must be given, not both")
errmsg = "No MySQL server available for group '{group}'"
servers = self.get_group_servers(group, use_cache=True)
if not servers:
raise InterfaceError(errmsg.format(group=group))
# Get the Master and return list (host, port, UUID)
primary = None
secondary = []
for server in servers:
if server.status == STATUS_SECONDARY:
secondary.append(server)
elif server.status == STATUS_PRIMARY:
primary = server
if mode in (MODE_WRITEONLY,
MODE_READWRITE) or status == STATUS_PRIMARY:
if not primary:
self.reset_cache(group=group)
raise InterfaceError((errmsg + ' {query}={value}').format(
query='status' if status else 'mode',
group=group,
value=status or mode))
return primary
# Return primary if no secondary is available
if not secondary and primary:
return primary
elif group in self._group_balancers:
next_secondary = self._group_balancers[group].get_next()[0]
for mysqlserver in secondary:
if next_secondary == mysqlserver.uuid:
return mysqlserver
self.reset_cache(group=group)
raise InterfaceError(errmsg.format(group=group, mode=mode))
def _xmlrpc_get_proxy(self):
"""Return the XMLRPC server proxy instance to MySQL Fabric
This method tries to get a valid connection to a MySQL Fabric
server.
Returns a XMLRPC ServerProxy instance.
"""
if self.is_connected:
return self._proxy
attempts = self._connect_attempts
delay = self._connect_delay
proxy = None
counter = 0
while counter != attempts:
counter += 1
try:
proxy = ServerProxy(self.uri)
proxy._some_nonexisting_method()
except Fault:
# We are actually connected
return proxy
except socket.error as exc:
if counter == attempts:
raise InterfaceError(
"Connection to MySQL Fabric failed ({0})".format(exc))
_LOGGER.debug(
"Retrying {host}:{port}, attempts {counter}".format(
host=self.host, port=self.port, counter=counter))
if delay > 0:
time.sleep(delay)
def connect(*args, **kwargs):
"""Create or get a MySQL connection object
In its simpliest form, Connect() will open a connection to a
MySQL server and return a MySQLConnection object.
When any connection pooling arguments are given, for example pool_name
or pool_size, a pool is created or a previously one is used to return
a PooledMySQLConnection.
Returns MySQLConnection or PooledMySQLConnection.
"""
if all(['fabric' in kwargs, 'failover' in kwargs]):
raise InterfaceError("fabric and failover arguments can not be used")
if 'fabric' in kwargs:
return mysql.connector.fabric.connect(*args, **kwargs)
# Failover
if 'failover' in kwargs:
return _get_failover_connection(**kwargs)
# Pooled connections
if any([key in kwargs for key in CNX_POOL_ARGS]):
return _get_pooled_connection(**kwargs)
# Regular connection
return MySQLConnection(*args, **kwargs)
def _get_pooled_connection(**kwargs):
"""Return a pooled MySQL connection"""
# If no pool name specified, generate one
from .pooling import (MySQLConPooL, generate_pool_name, CONNECTION_POOL_LOCK)
try:
pool_name = kwargs['pool_name']
except KeyError:
pool_name = generate_pool_name(**kwargs)
# Setup the pool, ensuring only 1 thread can update at a time
with CONNECTION_POOL_LOCK:
if pool_name not in _CONNECTION_POOLS:
_CONNECTION_POOLS[pool_name] = MySQLConPool(**kwargs)
elif isinstance(_CONNECTION_POOLS[pool_name], MySQLConPool):
# pool_size must be the same
check_size = _CONNECTION_POOLS[pool_name].pool_size
if ('pool_size' in kwargs
and kwargs['pool_size'] != check_size):
raise PoolError("Size can not be changed "
"for active pools.")
# Return pooled connection
try:
return _CONNECTION_POOLS[pool_name].get_connection()
except AttributeError:
raise InterfaceError(
"Failed getting connection from pool '{0}'".format(pool_name))
def get_shard_server(self, tables, key, scope=SCOPE_LOCAL, mode=None):
"""Get MySQL server information for a particular shard
Raises DatabaseError when the table is unknown or when tables are not
on the same shard. ValueError is raised when there is a problem
with the methods arguments. InterfaceError is raised for other errors.
"""
if not isinstance(tables, (list, tuple)):
raise ValueError("tables should be a sequence")
groups = []
for dbobj in tables:
try:
database, table = dbobj.split('.')
except ValueError:
raise ValueError(
"tables should be given as <database>.<table>, "
"was {0}".format(dbobj))
entry = self._cache.sharding_search(database, table)
if not entry:
self.get_sharding_information((table,), database)
entry = self._cache.sharding_search(database, table)
if not entry:
raise DatabaseError(
errno=errorcode.ER_BAD_TABLE_ERROR,
msg="Unknown table '{database}.{table}'".format(
database=database, table=table))
if scope == 'GLOBAL':
return self.get_group_server(entry.global_group, mode=mode)
if entry.shard_type == 'RANGE':
partitions = sorted(entry.partitioning.keys())
index = partitions[bisect(partitions, int(key)) - 1]
partition = entry.partitioning[index]
elif entry.shard_type == 'HASH':
md5key = md5(str(key))
partition_keys = sorted(
entry.partitioning.keys(), reverse=True)
index = partition_keys[-1]
for partkey in partition_keys:
if md5key.digest() >= b16decode(partkey):
index = partkey
break
partition = entry.partitioning[index]
else:
raise InterfaceError(
"Unsupported sharding type {0}".format(entry.shard_type))
groups.append(partition['group'])
if not all(group == groups[0] for group in groups):
raise DatabaseError(
"Tables are located in different shards.")
return self.get_group_server(groups[0], mode=mode)
def get_fabric_servers(self, fabric_cnx=None):
"""Get all MySQL Fabric instances
This method looks up the other MySQL Fabric instances which uses
the same metadata. The returned list contains dictionaries with
connection information such ass host and port. For example:
[
{'host': 'fabric_prod_1.example.com', 'port': 32274 },
{'host': 'fabric_prod_2.example.com', 'port': 32274 },
]
Returns a list of dictionaries
"""
inst = fabric_cnx or self.get_instance()
result = []
err_msg = "Looking up Fabric servers failed using {host}:{port}: {err}"
try:
(fabric_uuid_str, version, ttl,
addr_list) = inst.proxy.dump.fabric_nodes()
for addr in addr_list:
try:
host, port = addr.split(':', 2)
port = int(port)
except ValueError:
host, port = addr, MYSQL_FABRIC_PORT
result.append({'host': host, 'port': port})
except (Fault, socket.error) as exc:
msg = err_msg.format(err=str(exc), host=inst.host, port=inst.port)
raise InterfaceError(msg)
except (TypeError, AttributeError) as exc:
msg = err_msg.format(
err="No Fabric server available ({0})".format(exc),
host=inst.host,
port=inst.port)
raise InterfaceError(msg)
try:
fabric_uuid = uuid.UUID('{' + fabric_uuid_str + '}')
except TypeError:
fabric_uuid = uuid.uuid4()
return fabric_uuid, version, ttl, result
def request(self, host, handler, request_body, verbose):
uri = '{scheme}://{host}{handler}'.format(scheme=self._scheme,
host=host,
handler=handler)
if self._passmgr:
self._passmgr.add_password(None, uri, self._username,
self._password)
if self.verbose:
_LOGGER.debug("FabricTransport: {0}".format(uri))
opener = urllib2.build_opener(*self._handlers)
headers = {
'Content-Type': 'text/xml',
'User-Agent': self.user_agent,
}
req = urllib2.Request(uri, request_body, headers=headers)
try:
return self.parse_response(opener.open(req))
except urllib2.URLError as exc:
try:
if exc.code == 400:
reason = 'Permission denied'
else:
reason = exc.reason
msg = "{reason} ({code})".format(reason=reason, code=exc.code)
except AttributeError:
if 'SSL' in str(exc):
msg = "SSL error"
else:
msg = str(exc)
raise InterfaceError("Connection with Fabric failed: " + msg)
except BadStatusLine:
raise InterfaceError("Connection with Fabric failed: check SSL")
def get_sharding_information(self, tables=None, database=None):
"""Get and cache the sharding information for given tables
This method is fetching sharding information from MySQL Fabric
and caches the result. The tables argument must be sequence
of sequences contain the name of the database and table. If no
database is given, the value for the database argument will
be used.
Examples:
tables = [('salary',), ('employees',)]
get_sharding_information(tables, database='employees')
tables = [('salary', 'employees'), ('employees', employees)]
get_sharding_information(tables)
Raises InterfaceError on errors; ValueError when something is wrong
with the tables argument.
"""
if not isinstance(tables, (list, tuple)):
raise ValueError("tables should be a sequence")
patterns = []
for table in tables:
if not isinstance(table, (list, tuple)) and not database:
raise ValueError(
"No database specified for table {0}".format(table))
if isinstance(table, (list, tuple)):
dbase = table[1]
tbl = table[0]
else:
dbase = database
tbl = table
patterns.append("{0}.{1}".format(dbase, tbl))
inst = self.get_instance()
try:
result = inst.proxy.dump.sharding_information(
self._version_token, ','.join(patterns))
except (Fault, socket.error) as exc:
msg = "Looking up sharding information failed : {error}".format(
error=str(exc))
raise InterfaceError(msg)
for info in result[3]:
self._cache.sharding_cache_table(FabricShard(*info))
def seed(self, host=None, port=None):
"""Get MySQL Fabric Instances
This method uses host and port to connect to a MySQL Fabric server
and get all the instances managing the same metadata.
Raises InterfaceError on errors.
"""
host = host or self._init_host
port = port or self._init_port
fabinst = FabricConnection(self,
host,
port,
connect_attempts=self._connect_attempts,
connect_delay=self._connect_delay)
fabinst.connect()
fabric_uuid, version, ttl, fabrics = self.get_fabric_servers(fabinst)
if not fabrics:
# Raise, something went wrong.
raise InterfaceError("Failed getting list of Fabric servers")
if self._version_token == version:
return
_LOGGER.info("Loading Fabric configuration version {version}".format(
version=version))
self._fabric_uuid = fabric_uuid
self._version_token = version
if ttl > 0:
self._ttl = ttl
# Update the Fabric servers
for fabric in fabrics:
inst = FabricConnection(self,
fabric['host'],
fabric['port'],
connect_attempts=self._connect_attempts,
connect_delay=self._connect_delay)
inst_uuid = inst.uuid
if inst_uuid not in self._fabric_instances:
inst.connect()
self._fabric_instances[inst_uuid] = inst
_LOGGER.debug("Added new Fabric server {host}:{port}".format(
host=inst.host, port=inst.port))
def connect(*args, **kwargs):
"""Create or get a MySQL connection object
In its simpliest form, Connect() will open a connection to a
MySQL server and return a MySQLConnection object.
When any connection pooling arguments are given, for example pool_name
or pool_size, a pool is created or a previously one is used to return
a PooledMySQLConnection.
Returns MySQLConnection or PooledMySQLConnection.
"""
if 'fabric' in kwargs:
return mysql.connector.fabric.connect(*args, **kwargs)
# Pooled connections
if any([key in kwargs for key in CNX_POOL_ARGS]):
# If no pool name specified, generate one
try:
pool_name = kwargs['pool_name']
except KeyError:
pool_name = generate_pool_name(**kwargs)
# Setup the pool, ensuring only 1 thread can update at a time
with CONNECTION_POOL_LOCK:
if pool_name not in _CONNECTION_POOLS:
_CONNECTION_POOLS[pool_name] = MySQLConnectionPool(
*args, **kwargs)
elif isinstance(_CONNECTION_POOLS[pool_name], MySQLConnectionPool):
# pool_size must be the same
check_size = _CONNECTION_POOLS[pool_name].pool_size
if ('pool_size' in kwargs
and kwargs['pool_size'] != check_size):
raise PoolError("Size can not be changed "
"for active pools.")
# Return pooled connection
try:
return _CONNECTION_POOLS[pool_name].get_connection()
except AttributeError:
raise InterfaceError(
"Failed getting connection from pool '{0}'".format(pool_name))
# Regular connection
return MySQLConnection(*args, **kwargs)
def fetchall(self):
"""Returns all rows of a query result set
"""
if not self._have_unread_result():
from mysql.connector.errors import InterfaceError
raise InterfaceError(self.ERR_NO_RESULT_TO_FETCH)
(rows, eof) = self._connection.get_rows()
if self._nextrow[0]:
rows.insert(0, self._nextrow[0])
res = []
for row in rows:
res.append(self._row_to_python(row, self.description))
self._handle_eof(eof)
rowcount = len(rows)
if rowcount >= 0 and self._rowcount == -1:
self._rowcount = 0
self._rowcount += rowcount
return res
def get_group_servers(self, group, use_cache=True):
"""Get all MySQL servers in a group
This method returns information about all MySQL part of the
given high-availability group. When use_cache is set to
True, the cached information will be used.
Raises InterfaceError on errors.
Returns list of FabricMySQLServer objects.
"""
# Get group information from cache
if use_cache:
entry = self._cache.group_search(group)
if entry:
# Cache group information
_LOGGER.debug("Using cached group information")
return entry.servers
inst = self.get_instance()
result = []
try:
servers = inst.proxy.store.dump_servers(self._version_token,
group)[3]
except (Fault, socket.error) as exc:
msg = ("Looking up MySQL servers failed for group "
"{group}: {error}").format(error=str(exc), group=group)
raise InterfaceError(msg)
weights = []
for server in servers:
# We make sure, when using local groups, we skip the global group
if server[1] == group:
server[3] = int(server[3]) # port should be an int
mysqlserver = FabricMySQLServer(*server)
result.append(mysqlserver)
if mysqlserver.status == STATUS_SECONDARY:
weights.append((mysqlserver.uuid, mysqlserver.weight))
self._cache.cache_group(group, result)
if weights:
self._group_balancers[group] = WeightedRoundRobin(*weights)
return result
def _xmlrpc_get_proxy(self):
"""Return the XMLRPC server proxy instance to MySQL Fabric
This method tries to get a valid connection to a MySQL Fabric
server.
Returns a XMLRPC ServerProxy instance.
"""
if self.is_connected:
return self._proxy
attempts = self._connect_attempts
delay = self._connect_delay
proxy = None
counter = 0
while counter != attempts:
counter += 1
try:
if self._fabric.ssl_config:
scheme = 'https'
https_handler = FabricHTTPSHandler(self._fabric.ssl_config)
else:
https_handler = None
scheme = 'http'
transport = FabricTransport(self._fabric.username,
self._fabric.password,
verbose=0,
https_handler=https_handler)
proxy = ServerProxy(self.uri, transport=transport, verbose=0)
proxy._some_nonexisting_method() # pylint: disable=W0212
except Fault:
# We are actually connected
return proxy
except socket.error as exc:
if counter == attempts:
raise InterfaceError(
"Connection to MySQL Fabric failed ({0})".format(exc))
_LOGGER.debug(
"Retrying {host}:{port}, attempts {counter}".format(
host=self.host, port=self.port, counter=counter))
if delay > 0:
time.sleep(delay)
def get_instance(self):
"""Get a MySQL Fabric Instance
This method will get the next available MySQL Fabric Instance.
Raises InterfaceError when no instance is available or connected.
"""
nxt = 0
errmsg = "No MySQL Fabric instance available"
if not self._fabric_instances:
raise InterfaceError(errmsg + " (not seeded?)")
if sys.version_info[0] == 2:
instance_list = self._fabric_instances.keys()
inst = self._fabric_instances[instance_list[nxt]]
else:
inst = self._fabric_instances[list(self._fabric_instances)[nxt]]
if not inst.is_connected:
inst.connect()
return inst
def _get_failover_connection(**kwargs):
"""Return a MySQL connection and try to failover if needed
An InterfaceError is raise when no MySQL is available. ValueError is
raised when the failover server configuration contains an illegal
connection argument. Supported arguments are user, password, host, port,
unix_socket and database. ValueError is also raised when the failover
argument was not provided.
Returns MySQLConnection instance.
"""
config = kwargs.copy()
try:
failover = config['failover']
except KeyError:
raise ValueError('failover argument not provided')
del config['failover']
support_cnx_args = set([
'user', 'password', 'host', 'port', 'unix_socket', 'database',
'pool_name', 'pool_size'
])
# First check if we can add all use the configuration
for server in failover:
diff = set(server.keys()) - support_cnx_args
if diff:
raise ValueError(
"Unsupported connection argument {0} in failover: {1}".format(
's' if len(diff) > 1 else '', ', '.join(diff)))
for server in failover:
new_config = config.copy()
new_config.update(server)
try:
return connect(**new_config)
except Error:
# If we failed to connect, we try the next server
pass
raise InterfaceError("Could not failover: no MySQL server available")
def connect(*args, **kwargs):
"""Create or get a MySQL connection object
In its simpliest form, Connect() will open a connection to a
MySQL server and return a AioSQLConnection object.
When any connection pooling arguments are given, for example pool_name
or pool_size, a pool is created or a previously one is used to return
a PooledAioSQLConnection.
Returns AioSQLConnection or PooledAioSQLConnection.
"""
# Option files
if 'option_files' in kwargs:
new_config = read_option_files(**kwargs)
return connect(**new_config)
if all(['fabric' in kwargs, 'failover' in kwargs]):
raise InterfaceError("fabric and failover arguments can not be used")
if 'fabric' in kwargs:
from .fabric import connect as fabric_connect
return fabric_connect(*args, **kwargs)
# Failover
if 'failover' in kwargs:
return _get_failover_connection(**kwargs)
# Pooled connections
try:
from mysql.connector.pooling import CNX_POOL_ARGS
if any([key in kwargs for key in CNX_POOL_ARGS]):
return _get_pooled_connection(**kwargs)
except NameError:
# No pooling
pass
# Regular connection
return AioMySQLConnection(*args, **kwargs)
def read_packet(self):
"""Read a MySQL packet from the socket
This method reads a MySQL packet form the socket, parses it and
returns the type and the payload. The type is the value of the
first byte of the payload.
:return: Tuple with type and payload of packet.
:rtype: tuple
"""
header = bytearray(b'')
header_len = 0
while header_len < 4:
chunk = self.request.recv(4 - header_len)
if not chunk:
raise InterfaceError(errno=2013)
header += chunk
header_len = len(header)
length = struct.unpack_from('<I', buffer(header[0:3] + b'\x00'))[0]
self._curr_pktnr = header[-1]
data = bytearray(self.request.recv(length))
return data[0], data[1:]
def _connect(self):
"""Get a MySQL server based on properties and connect
This method gets a MySQL server from MySQL Fabric using already
properties set using the set_property() method. You can specify how
many times and the delay between trying using attempts and
attempt_delay.
Raises InterfaceError on errors. A ValueError is raised when
both group and sharding are specified, or neither of them.
"""
if self.is_connected():
return
props = self._cnx_properties
attempts = props['attempts']
attempt_delay = props['attempt_delay']
if props['group'] and props['tables']:
raise ValueError(
"Either 'group' or 'tables' property can be set, not both.")
if not (props['group'] or props['tables']):
raise ValueError(
"Either 'group' or 'tables' property needs to be set.")
dbconfig = self._mysql_config.copy()
counter = 0
while counter != attempts:
counter += 1
try:
group = None
if props['tables']:
if props['scope'] == 'LOCAL' and not props['key']:
raise ValueError(
"Scope 'LOCAL' needs key property to be set")
mysqlserver = self._fabric.get_shard_server(
props['tables'],
props['key'],
scope=props['scope'],
mode=props['mode'])
elif props['group']:
group = props['group']
mysqlserver = self._fabric.get_group_server(
group, mode=props['mode'])
else:
raise InterfaceError(
"Missing group or key and tables properties")
except InterfaceError as err:
_LOGGER.debug(
"Trying to get MySQL server (attempt {0}; {1})".format(
counter, err))
if counter == attempts:
raise InterfaceError(
"Error getting connection: {0}".format(err))
if attempt_delay > 0:
time.sleep(attempt_delay)
continue
# Make sure we do not change the stored configuration
dbconfig['host'] = mysqlserver.host
dbconfig['port'] = mysqlserver.port
try:
self._mysql_cnx = mysql.connector.connect(**dbconfig)
except InterfaceError as err:
self._fabric.set_server_status(mysqlserver.uuid, STATUS_FAULTY)
self.reset_cache(mysqlserver.group)
if counter == attempts:
raise InterfaceError(
"Reported faulty server to Fabric ({0})".format(err))
else:
self._fabric_mysql_server = mysqlserver
break
def _connect(self):
"""Get a MySQL server based on properties and connect
This method gets a MySQL server from MySQL Fabric using already
properties set using the set_property() method. You can specify how
many times and the delay between trying using attempts and
attempt_delay.
Raises ValueError when there are problems with arguments or
properties; InterfaceError on connectivity errors.
"""
if self.is_connected():
return
props = self._cnx_properties
attempts = props['attempts']
attempt_delay = props['attempt_delay']
dbconfig = self._mysql_config.copy()
counter = 0
while counter != attempts:
counter += 1
try:
group = None
if props['tables']:
if props['scope'] == 'LOCAL' and not props['key']:
raise ValueError(
"Scope 'LOCAL' needs key property to be set")
mysqlserver = self._fabric.get_shard_server(
props['tables'],
props['key'],
scope=props['scope'],
mode=props['mode'])
elif props['group']:
group = props['group']
mysqlserver = self._fabric.get_group_server(
group, mode=props['mode'])
else:
raise ValueError(
"Missing group or key and tables properties")
except InterfaceError as exc:
_LOGGER.debug(
"Trying to get MySQL server (attempt {0}; {1})".format(
counter, exc))
if counter == attempts:
raise InterfaceError(
"Error getting connection: {0}".format(exc))
if attempt_delay > 0:
_LOGGER.debug("Waiting {0}".format(attempt_delay))
time.sleep(attempt_delay)
continue
# Make sure we do not change the stored configuration
dbconfig['host'] = mysqlserver.host
dbconfig['port'] = mysqlserver.port
try:
self._mysql_cnx = mysql.connector.connect(**dbconfig)
except Error as exc:
if counter == attempts:
self.reset_cache(mysqlserver.group)
self._fabric.report_failure(mysqlserver.uuid, exc.errno)
raise InterfaceError(
"Reported faulty server to Fabric ({0})".format(exc))
if attempt_delay > 0:
time.sleep(attempt_delay)
continue
else:
self._fabric_mysql_server = mysqlserver
break
