def __init__(self,
host,
username=None,
password=None,
port=MYSQL_FABRIC_PORT,
connect_attempts=_CNX_ATTEMPT_MAX,
connect_delay=_CNX_ATTEMPT_DELAY,
report_errors=False,
ssl_ca=None,
ssl_key=None,
ssl_cert=None,
user=None):
"""Initialize"""
self._fabric_instances = {}
self._fabric_uuid = None
self._ttl = 1 * 60 # one minute by default
self._version_token = None
self._connect_attempts = connect_attempts
self._connect_delay = connect_delay
self._cache = FabricCache()
self._group_balancers = {}
self._init_host = host
self._init_port = port
self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert)
self._report_errors = report_errors
if user and username:
raise ValueError("can not specify both user and username")
self._username = user or username
self._password = password
def reset_cache(self, group=None):
"""Reset cached information
This method destroys all cached information.
"""
if group:
_LOGGER.debug(
"Resetting cache for group '{group}'".format(group=group))
self.get_group_servers(group, use_cache=False)
else:
_LOGGER.debug("Resetting cache")
self._cache = FabricCache()
def reset_cache(self, group=None):
"""Reset cached information
This method destroys all cached information.
"""
if group:
_LOGGER.debug("Resetting cache for group '{group}'".format(
group=group))
self.get_group_servers(group, use_cache=False)
else:
_LOGGER.debug("Resetting cache")
self._cache = FabricCache()
def __init__(self, host, username=None, password=None,
port=MYSQL_FABRIC_PORT,
connect_attempts=_CNX_ATTEMPT_MAX,
connect_delay=_CNX_ATTEMPT_DELAY,
report_errors=False,
ssl_ca=None, ssl_key=None, ssl_cert=None, user=None):
"""Initialize"""
self._fabric_instances = {}
self._fabric_uuid = None
self._ttl = 1 * 60 # one minute by default
self._version_token = None
self._connect_attempts = connect_attempts
self._connect_delay = connect_delay
self._cache = FabricCache()
self._group_balancers = {}
self._init_host = host
self._init_port = port
self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert)
self._report_errors = report_errors
if user and username:
raise ValueError("can not specify both user and username")
self._username = user or username
self._password = password
class Fabric(object):
"""Class managing MySQL Fabric instances"""
def __init__(self, host, username=None, password=None,
port=MYSQL_FABRIC_PORT,
connect_attempts=_CNX_ATTEMPT_MAX,
connect_delay=_CNX_ATTEMPT_DELAY,
report_errors=False,
ssl_ca=None, ssl_key=None, ssl_cert=None, user=None):
"""Initialize"""
self._fabric_instances = {}
self._fabric_uuid = None
self._ttl = 1 * 60 # one minute by default
self._version_token = None
self._connect_attempts = connect_attempts
self._connect_delay = connect_delay
self._cache = FabricCache()
self._group_balancers = {}
self._init_host = host
self._init_port = port
self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert)
self._report_errors = report_errors
if user and username:
raise ValueError("can not specify both user and username")
self._username = user or username
self._password = password
@property
def username(self):
return self._username
@property
def password(self):
return self._password
@property
def ssl_config(self):
return self._ssl
def seed(self, host=None, port=None):
"""Get MySQL Fabric Instances
This method uses host and port to connect to a MySQL Fabric server
and get all the instances managing the same metadata.
Raises InterfaceError on errors.
"""
host = host or self._init_host
port = port or self._init_port
fabinst = FabricConnection(self, host, port,
connect_attempts=self._connect_attempts,
connect_delay=self._connect_delay)
fabinst.connect()
fabric_uuid, version, ttl, fabrics = self.get_fabric_servers(
fabinst)
if not fabrics:
# Raise, something went wrong.
raise InterfaceError("Failed getting list of Fabric servers")
if self._version_token == version:
return
_LOGGER.info(
"Loading Fabric configuration version {version}".format(
version=version))
self._fabric_uuid = fabric_uuid
self._version_token = version
if ttl > 0:
self._ttl = ttl
# Update the Fabric servers
for fabric in fabrics:
inst = FabricConnection(self, fabric['host'], fabric['port'],
connect_attempts=self._connect_attempts,
connect_delay=self._connect_delay)
inst_uuid = inst.uuid
if inst_uuid not in self._fabric_instances:
inst.connect()
self._fabric_instances[inst_uuid] = inst
_LOGGER.debug(
"Added new Fabric server {host}:{port}".format(
host=inst.host, port=inst.port))
def reset_cache(self, group=None):
"""Reset cached information
This method destroys all cached information.
"""
if group:
_LOGGER.debug("Resetting cache for group '{group}'".format(
group=group))
self.get_group_servers(group, use_cache=False)
else:
_LOGGER.debug("Resetting cache")
self._cache = FabricCache()
def get_instance(self):
"""Get a MySQL Fabric Instance
This method will get the next available MySQL Fabric Instance.
Raises InterfaceError when no instance is available or connected.
"""
nxt = 0
errmsg = "No MySQL Fabric instance available"
if not self._fabric_instances:
raise InterfaceError(errmsg + " (not seeded?)")
if sys.version_info[0] == 2:
instance_list = self._fabric_instances.keys()
inst = self._fabric_instances[instance_list[nxt]]
else:
inst = self._fabric_instances[list(self._fabric_instances)[nxt]]
if not inst.is_connected:
inst.connect()
return inst
def report_failure(self, server_uuid, errno):
"""Report failure to Fabric
This method sets the status of a MySQL server identified by
server_uuid.
"""
if not self._report_errors:
return
errno = int(errno)
current_host = socket.getfqdn()
if errno in REPORT_ERRORS or errno in REPORT_ERRORS_EXTRA:
_LOGGER.debug("Reporting error %d of server %s", errno,
server_uuid)
inst = self.get_instance()
try:
inst.proxy.threat.report_failure(server_uuid, current_host,
errno)
except (Fault, socket.error) as exc:
_LOGGER.debug("Failed reporting server to Fabric.")
# Not requiring further action
def get_fabric_servers(self, fabric_cnx=None):
"""Get all MySQL Fabric instances
This method looks up the other MySQL Fabric instances which uses
the same metadata. The returned list contains dictionaries with
connection information such ass host and port. For example:
[
{'host': 'fabric_prod_1.example.com', 'port': 32274 },
{'host': 'fabric_prod_2.example.com', 'port': 32274 },
]
Returns a list of dictionaries
"""
inst = fabric_cnx or self.get_instance()
result = []
err_msg = "Looking up Fabric servers failed using {host}:{port}: {err}"
try:
(fabric_uuid_str, version,
ttl, addr_list) = inst.proxy.dump.fabric_nodes()
for addr in addr_list:
try:
host, port = addr.split(':', 2)
port = int(port)
except ValueError:
host, port = addr, MYSQL_FABRIC_PORT
result.append({'host': host, 'port': port})
except (Fault, socket.error) as exc:
msg = err_msg.format(err=str(exc), host=inst.host, port=inst.port)
raise InterfaceError(msg)
except (TypeError, AttributeError) as exc:
msg = err_msg.format(
err="No Fabric server available ({0})".format(exc),
host=inst.host, port=inst.port)
raise InterfaceError(msg)
try:
fabric_uuid = uuid.UUID('{' + fabric_uuid_str + '}')
except TypeError:
fabric_uuid = uuid.uuid4()
return fabric_uuid, version, ttl, result
def get_group_servers(self, group, use_cache=True):
"""Get all MySQL servers in a group
This method returns information about all MySQL part of the
given high-availability group. When use_cache is set to
True, the cached information will be used.
Raises InterfaceError on errors.
Returns list of FabricMySQLServer objects.
"""
# Get group information from cache
if use_cache:
entry = self._cache.group_search(group)
if entry:
# Cache group information
return entry.servers
inst = self.get_instance()
result = []
try:
servers = inst.proxy.dump.servers(
self._version_token, group)[3]
except (Fault, socket.error) as exc:
msg = ("Looking up MySQL servers failed for group "
"{group}: {error}").format(error=str(exc), group=group)
raise InterfaceError(msg)
weights = []
for server in servers:
# We make sure, when using local groups, we skip the global group
if server[1] == group:
server[3] = int(server[3]) # port should be an int
mysqlserver = FabricMySQLServer(*server)
result.append(mysqlserver)
if mysqlserver.status == STATUS_SECONDARY:
weights.append((mysqlserver.uuid, mysqlserver.weight))
self._cache.cache_group(group, result)
if weights:
self._group_balancers[group] = WeightedRoundRobin(*weights)
return result
def get_group_server(self, group, mode=None, status=None):
"""Get a MySQL server from a group
The method uses MySQL Fabric to get the correct MySQL server
for the specified group. You can specify mode or status, but
not both.
The mode argument will decide whether the primary or a secondary
server is returned. When no secondary server is available, the
primary is returned.
Status is used to force getting either a primary or a secondary.
The returned tuple contains host, port and uuid.
Raises InterfaceError on errors; ValueError when both mode
and status are given.
Returns a FabricMySQLServer object.
"""
if mode and status:
raise ValueError(
"Either mode or status must be given, not both")
errmsg = "No MySQL server available for group '{group}'"
servers = self.get_group_servers(group, use_cache=True)
if not servers:
raise InterfaceError(errmsg.format(group=group))
# Get the Master and return list (host, port, UUID)
primary = None
secondary = []
for server in servers:
if server.status == STATUS_SECONDARY:
secondary.append(server)
elif server.status == STATUS_PRIMARY:
primary = server
if mode in (MODE_WRITEONLY, MODE_READWRITE) or status == STATUS_PRIMARY:
if not primary:
self.reset_cache(group=group)
raise InterfaceError((errmsg + ' {query}={value}').format(
query='status' if status else 'mode',
group=group,
value=status or mode))
return primary
# Return primary if no secondary is available
if not secondary and primary:
return primary
elif group in self._group_balancers:
next_secondary = self._group_balancers[group].get_next()[0]
for mysqlserver in secondary:
if next_secondary == mysqlserver.uuid:
return mysqlserver
self.reset_cache(group=group)
raise InterfaceError(errmsg.format(group=group, mode=mode))
def get_sharding_information(self, tables=None, database=None):
"""Get and cache the sharding information for given tables
This method is fetching sharding information from MySQL Fabric
and caches the result. The tables argument must be sequence
of sequences contain the name of the database and table. If no
database is given, the value for the database argument will
be used.
Examples:
tables = [('salary',), ('employees',)]
get_sharding_information(tables, database='employees')
tables = [('salary', 'employees'), ('employees', employees)]
get_sharding_information(tables)
Raises InterfaceError on errors; ValueError when something is wrong
with the tables argument.
"""
if not isinstance(tables, (list, tuple)):
raise ValueError("tables should be a sequence")
patterns = []
for table in tables:
if not isinstance(table, (list, tuple)) and not database:
raise ValueError("No database specified for table {0}".format(
table))
if isinstance(table, (list, tuple)):
dbase = table[1]
tbl = table[0]
else:
dbase = database
tbl = table
patterns.append("{0}.{1}".format(dbase, tbl))
inst = self.get_instance()
try:
result = inst.proxy.dump.sharding_information(
self._version_token, ','.join(patterns))
except (Fault, socket.error) as exc:
msg = "Looking up sharding information failed : {error}".format(
error=str(exc))
raise InterfaceError(msg)
for info in result[3]:
self._cache.sharding_cache_table(FabricShard(*info))
def get_shard_server(self, tables, key, scope=SCOPE_LOCAL, mode=None):
"""Get MySQL server information for a particular shard
Raises DatabaseError when the table is unknown or when tables are not
on the same shard. ValueError is raised when there is a problem
with the methods arguments. InterfaceError is raised for other errors.
"""
if not isinstance(tables, (list, tuple)):
raise ValueError("tables should be a sequence")
groups = []
for dbobj in tables:
try:
database, table = dbobj.split('.')
except ValueError:
raise ValueError(
"tables should be given as <database>.<table>, "
"was {0}".format(dbobj))
entry = self._cache.sharding_search(database, table)
if not entry:
self.get_sharding_information((table,), database)
entry = self._cache.sharding_search(database, table)
if not entry:
raise DatabaseError(
errno=errorcode.ER_BAD_TABLE_ERROR,
msg="Unknown table '{database}.{table}'".format(
database=database, table=table))
if scope == 'GLOBAL':
return self.get_group_server(entry.global_group, mode=mode)
if entry.shard_type == 'RANGE':
partitions = sorted(entry.partitioning.keys())
index = partitions[bisect(partitions, int(key)) - 1]
partition = entry.partitioning[index]
elif entry.shard_type == 'HASH':
md5key = md5(str(key))
partition_keys = sorted(
entry.partitioning.keys(), reverse=True)
index = partition_keys[-1]
for partkey in partition_keys:
if md5key.digest() >= b16decode(partkey):
index = partkey
break
partition = entry.partitioning[index]
else:
raise InterfaceError(
"Unsupported sharding type {0}".format(entry.shard_type))
groups.append(partition['group'])
if not all(group == groups[0] for group in groups):
raise DatabaseError(
"Tables are located in different shards.")
return self.get_group_server(groups[0], mode=mode)
class Fabric(object):
"""Class managing MySQL Fabric instances"""
def __init__(self,
host,
username=None,
password=None,
port=MYSQL_FABRIC_PORT,
connect_attempts=_CNX_ATTEMPT_MAX,
connect_delay=_CNX_ATTEMPT_DELAY,
report_errors=False,
ssl_ca=None,
ssl_key=None,
ssl_cert=None,
user=None):
"""Initialize"""
self._fabric_instances = {}
self._fabric_uuid = None
self._ttl = 1 * 60 # one minute by default
self._version_token = None
self._connect_attempts = connect_attempts
self._connect_delay = connect_delay
self._cache = FabricCache()
self._group_balancers = {}
self._init_host = host
self._init_port = port
self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert)
self._report_errors = report_errors
if user and username:
raise ValueError("can not specify both user and username")
self._username = user or username
self._password = password
@property
def username(self):
return self._username
@property
def password(self):
return self._password
@property
def ssl_config(self):
return self._ssl
def seed(self, host=None, port=None):
"""Get MySQL Fabric Instances
This method uses host and port to connect to a MySQL Fabric server
and get all the instances managing the same metadata.
Raises InterfaceError on errors.
"""
host = host or self._init_host
port = port or self._init_port
fabinst = FabricConnection(self,
host,
port,
connect_attempts=self._connect_attempts,
connect_delay=self._connect_delay)
fabinst.connect()
fabric_uuid, version, ttl, fabrics = self.get_fabric_servers(fabinst)
if not fabrics:
# Raise, something went wrong.
raise InterfaceError("Failed getting list of Fabric servers")
if self._version_token == version:
return
_LOGGER.info("Loading Fabric configuration version {version}".format(
version=version))
self._fabric_uuid = fabric_uuid
self._version_token = version
if ttl > 0:
self._ttl = ttl
# Update the Fabric servers
for fabric in fabrics:
inst = FabricConnection(self,
fabric['host'],
fabric['port'],
connect_attempts=self._connect_attempts,
connect_delay=self._connect_delay)
inst_uuid = inst.uuid
if inst_uuid not in self._fabric_instances:
inst.connect()
self._fabric_instances[inst_uuid] = inst
_LOGGER.debug("Added new Fabric server {host}:{port}".format(
host=inst.host, port=inst.port))
def reset_cache(self, group=None):
"""Reset cached information
This method destroys all cached information.
"""
if group:
_LOGGER.debug(
"Resetting cache for group '{group}'".format(group=group))
self.get_group_servers(group, use_cache=False)
else:
_LOGGER.debug("Resetting cache")
self._cache = FabricCache()
def get_instance(self):
"""Get a MySQL Fabric Instance
This method will get the next available MySQL Fabric Instance.
Raises InterfaceError when no instance is available or connected.
"""
nxt = 0
errmsg = "No MySQL Fabric instance available"
if not self._fabric_instances:
raise InterfaceError(errmsg + " (not seeded?)")
if sys.version_info[0] == 2:
instance_list = self._fabric_instances.keys()
inst = self._fabric_instances[instance_list[nxt]]
else:
inst = self._fabric_instances[list(self._fabric_instances)[nxt]]
if not inst.is_connected:
inst.connect()
return inst
def report_failure(self, server_uuid, errno):
"""Report failure to Fabric
This method sets the status of a MySQL server identified by
server_uuid.
"""
if not self._report_errors:
return
errno = int(errno)
current_host = socket.getfqdn()
if errno in REPORT_ERRORS or errno in REPORT_ERRORS_EXTRA:
_LOGGER.debug("Reporting error %d of server %s", errno,
server_uuid)
inst = self.get_instance()
try:
inst.proxy.threat.report_failure(server_uuid, current_host,
errno)
except (Fault, socket.error) as exc:
_LOGGER.debug("Failed reporting server to Fabric.")
# Not requiring further action
def get_fabric_servers(self, fabric_cnx=None):
"""Get all MySQL Fabric instances
This method looks up the other MySQL Fabric instances which uses
the same metadata. The returned list contains dictionaries with
connection information such ass host and port. For example:
[
{'host': 'fabric_prod_1.example.com', 'port': 32274 },
{'host': 'fabric_prod_2.example.com', 'port': 32274 },
]
Returns a list of dictionaries
"""
inst = fabric_cnx or self.get_instance()
result = []
err_msg = "Looking up Fabric servers failed using {host}:{port}: {err}"
try:
(fabric_uuid_str, version, ttl,
addr_list) = inst.proxy.dump.fabric_nodes()
for addr in addr_list:
try:
host, port = addr.split(':', 2)
port = int(port)
except ValueError:
host, port = addr, MYSQL_FABRIC_PORT
result.append({'host': host, 'port': port})
except (Fault, socket.error) as exc:
msg = err_msg.format(err=str(exc), host=inst.host, port=inst.port)
raise InterfaceError(msg)
except (TypeError, AttributeError) as exc:
msg = err_msg.format(
err="No Fabric server available ({0})".format(exc),
host=inst.host,
port=inst.port)
raise InterfaceError(msg)
try:
fabric_uuid = uuid.UUID('{' + fabric_uuid_str + '}')
except TypeError:
fabric_uuid = uuid.uuid4()
return fabric_uuid, version, ttl, result
def get_group_servers(self, group, use_cache=True):
"""Get all MySQL servers in a group
This method returns information about all MySQL part of the
given high-availability group. When use_cache is set to
True, the cached information will be used.
Raises InterfaceError on errors.
Returns list of FabricMySQLServer objects.
"""
# Get group information from cache
if use_cache:
entry = self._cache.group_search(group)
if entry:
# Cache group information
return entry.servers
inst = self.get_instance()
result = []
try:
servers = inst.proxy.dump.servers(self._version_token, group)[3]
except (Fault, socket.error) as exc:
msg = ("Looking up MySQL servers failed for group "
"{group}: {error}").format(error=str(exc), group=group)
raise InterfaceError(msg)
weights = []
for server in servers:
# We make sure, when using local groups, we skip the global group
if server[1] == group:
server[3] = int(server[3]) # port should be an int
mysqlserver = FabricMySQLServer(*server)
result.append(mysqlserver)
if mysqlserver.status == STATUS_SECONDARY:
weights.append((mysqlserver.uuid, mysqlserver.weight))
self._cache.cache_group(group, result)
if weights:
self._group_balancers[group] = WeightedRoundRobin(*weights)
return result
def get_group_server(self, group, mode=None, status=None):
"""Get a MySQL server from a group
The method uses MySQL Fabric to get the correct MySQL server
for the specified group. You can specify mode or status, but
not both.
The mode argument will decide whether the primary or a secondary
server is returned. When no secondary server is available, the
primary is returned.
Status is used to force getting either a primary or a secondary.
The returned tuple contains host, port and uuid.
Raises InterfaceError on errors; ValueError when both mode
and status are given.
Returns a FabricMySQLServer object.
"""
if mode and status:
raise ValueError("Either mode or status must be given, not both")
errmsg = "No MySQL server available for group '{group}'"
servers = self.get_group_servers(group, use_cache=True)
if not servers:
raise InterfaceError(errmsg.format(group=group))
# Get the Master and return list (host, port, UUID)
primary = None
secondary = []
for server in servers:
if server.status == STATUS_SECONDARY:
secondary.append(server)
elif server.status == STATUS_PRIMARY:
primary = server
if mode in (MODE_WRITEONLY,
MODE_READWRITE) or status == STATUS_PRIMARY:
if not primary:
self.reset_cache(group=group)
raise InterfaceError((errmsg + ' {query}={value}').format(
query='status' if status else 'mode',
group=group,
value=status or mode))
return primary
# Return primary if no secondary is available
if not secondary and primary:
return primary
elif group in self._group_balancers:
next_secondary = self._group_balancers[group].get_next()[0]
for mysqlserver in secondary:
if next_secondary == mysqlserver.uuid:
return mysqlserver
self.reset_cache(group=group)
raise InterfaceError(errmsg.format(group=group, mode=mode))
def get_sharding_information(self, tables=None, database=None):
"""Get and cache the sharding information for given tables
This method is fetching sharding information from MySQL Fabric
and caches the result. The tables argument must be sequence
of sequences contain the name of the database and table. If no
database is given, the value for the database argument will
be used.
Examples:
tables = [('salary',), ('employees',)]
get_sharding_information(tables, database='employees')
tables = [('salary', 'employees'), ('employees', employees)]
get_sharding_information(tables)
Raises InterfaceError on errors; ValueError when something is wrong
with the tables argument.
"""
if not isinstance(tables, (list, tuple)):
raise ValueError("tables should be a sequence")
patterns = []
for table in tables:
if not isinstance(table, (list, tuple)) and not database:
raise ValueError(
"No database specified for table {0}".format(table))
if isinstance(table, (list, tuple)):
dbase = table[1]
tbl = table[0]
else:
dbase = database
tbl = table
patterns.append("{0}.{1}".format(dbase, tbl))
inst = self.get_instance()
try:
result = inst.proxy.dump.sharding_information(
self._version_token, ','.join(patterns))
except (Fault, socket.error) as exc:
msg = "Looking up sharding information failed : {error}".format(
error=str(exc))
raise InterfaceError(msg)
for info in result[3]:
self._cache.sharding_cache_table(FabricShard(*info))
def get_shard_server(self, tables, key, scope=SCOPE_LOCAL, mode=None):
"""Get MySQL server information for a particular shard
Raises DatabaseError when the table is unknown or when tables are not
on the same shard. ValueError is raised when there is a problem
with the methods arguments. InterfaceError is raised for other errors.
"""
if not isinstance(tables, (list, tuple)):
raise ValueError("tables should be a sequence")
groups = []
for dbobj in tables:
try:
database, table = dbobj.split('.')
except ValueError:
raise ValueError(
"tables should be given as <database>.<table>, "
"was {0}".format(dbobj))
entry = self._cache.sharding_search(database, table)
if not entry:
self.get_sharding_information((table, ), database)
entry = self._cache.sharding_search(database, table)
if not entry:
raise DatabaseError(
errno=errorcode.ER_BAD_TABLE_ERROR,
msg="Unknown table '{database}.{table}'".format(
database=database, table=table))
if scope == 'GLOBAL':
return self.get_group_server(entry.global_group, mode=mode)
if entry.shard_type == 'RANGE':
partitions = sorted(entry.partitioning.keys())
index = partitions[bisect(partitions, int(key)) - 1]
partition = entry.partitioning[index]
elif entry.shard_type == 'HASH':
md5key = md5(str(key))
partition_keys = sorted(entry.partitioning.keys(),
reverse=True)
index = partition_keys[-1]
for partkey in partition_keys:
if md5key.digest() >= b16decode(partkey):
index = partkey
break
partition = entry.partitioning[index]
else:
raise InterfaceError("Unsupported sharding type {0}".format(
entry.shard_type))
groups.append(partition['group'])
if not all(group == groups[0] for group in groups):
raise DatabaseError("Tables are located in different shards.")
return self.get_group_server(groups[0], mode=mode)