def __init__(self, host, username=None, password=None, port=MYSQL_FABRIC_PORT, connect_attempts=_CNX_ATTEMPT_MAX, connect_delay=_CNX_ATTEMPT_DELAY, report_errors=False, ssl_ca=None, ssl_key=None, ssl_cert=None, user=None): """Initialize""" self._fabric_instances = {} self._fabric_uuid = None self._ttl = 1 * 60 # one minute by default self._version_token = None self._connect_attempts = connect_attempts self._connect_delay = connect_delay self._cache = FabricCache() self._group_balancers = {} self._init_host = host self._init_port = port self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert) self._report_errors = report_errors if user and username: raise ValueError("can not specify both user and username") self._username = user or username self._password = password
def reset_cache(self, group=None): """Reset cached information This method destroys all cached information. """ if group: _LOGGER.debug( "Resetting cache for group '{group}'".format(group=group)) self.get_group_servers(group, use_cache=False) else: _LOGGER.debug("Resetting cache") self._cache = FabricCache()
def reset_cache(self, group=None): """Reset cached information This method destroys all cached information. """ if group: _LOGGER.debug("Resetting cache for group '{group}'".format( group=group)) self.get_group_servers(group, use_cache=False) else: _LOGGER.debug("Resetting cache") self._cache = FabricCache()
class Fabric(object): """Class managing MySQL Fabric instances""" def __init__(self, host, username=None, password=None, port=MYSQL_FABRIC_PORT, connect_attempts=_CNX_ATTEMPT_MAX, connect_delay=_CNX_ATTEMPT_DELAY, report_errors=False, ssl_ca=None, ssl_key=None, ssl_cert=None, user=None): """Initialize""" self._fabric_instances = {} self._fabric_uuid = None self._ttl = 1 * 60 # one minute by default self._version_token = None self._connect_attempts = connect_attempts self._connect_delay = connect_delay self._cache = FabricCache() self._group_balancers = {} self._init_host = host self._init_port = port self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert) self._report_errors = report_errors if user and username: raise ValueError("can not specify both user and username") self._username = user or username self._password = password @property def username(self): return self._username @property def password(self): return self._password @property def ssl_config(self): return self._ssl def seed(self, host=None, port=None): """Get MySQL Fabric Instances This method uses host and port to connect to a MySQL Fabric server and get all the instances managing the same metadata. Raises InterfaceError on errors. """ host = host or self._init_host port = port or self._init_port fabinst = FabricConnection(self, host, port, connect_attempts=self._connect_attempts, connect_delay=self._connect_delay) fabinst.connect() fabric_uuid, version, ttl, fabrics = self.get_fabric_servers( fabinst) if not fabrics: # Raise, something went wrong. raise InterfaceError("Failed getting list of Fabric servers") if self._version_token == version: return _LOGGER.info( "Loading Fabric configuration version {version}".format( version=version)) self._fabric_uuid = fabric_uuid self._version_token = version if ttl > 0: self._ttl = ttl # Update the Fabric servers for fabric in fabrics: inst = FabricConnection(self, fabric['host'], fabric['port'], connect_attempts=self._connect_attempts, connect_delay=self._connect_delay) inst_uuid = inst.uuid if inst_uuid not in self._fabric_instances: inst.connect() self._fabric_instances[inst_uuid] = inst _LOGGER.debug( "Added new Fabric server {host}:{port}".format( host=inst.host, port=inst.port)) def reset_cache(self, group=None): """Reset cached information This method destroys all cached information. """ if group: _LOGGER.debug("Resetting cache for group '{group}'".format( group=group)) self.get_group_servers(group, use_cache=False) else: _LOGGER.debug("Resetting cache") self._cache = FabricCache() def get_instance(self): """Get a MySQL Fabric Instance This method will get the next available MySQL Fabric Instance. Raises InterfaceError when no instance is available or connected. """ nxt = 0 errmsg = "No MySQL Fabric instance available" if not self._fabric_instances: raise InterfaceError(errmsg + " (not seeded?)") if sys.version_info[0] == 2: instance_list = self._fabric_instances.keys() inst = self._fabric_instances[instance_list[nxt]] else: inst = self._fabric_instances[list(self._fabric_instances)[nxt]] if not inst.is_connected: inst.connect() return inst def report_failure(self, server_uuid, errno): """Report failure to Fabric This method sets the status of a MySQL server identified by server_uuid. """ if not self._report_errors: return errno = int(errno) current_host = socket.getfqdn() if errno in REPORT_ERRORS or errno in REPORT_ERRORS_EXTRA: _LOGGER.debug("Reporting error %d of server %s", errno, server_uuid) inst = self.get_instance() try: inst.proxy.threat.report_failure(server_uuid, current_host, errno) except (Fault, socket.error) as exc: _LOGGER.debug("Failed reporting server to Fabric.") # Not requiring further action def get_fabric_servers(self, fabric_cnx=None): """Get all MySQL Fabric instances This method looks up the other MySQL Fabric instances which uses the same metadata. The returned list contains dictionaries with connection information such ass host and port. For example: [ {'host': 'fabric_prod_1.example.com', 'port': 32274 }, {'host': 'fabric_prod_2.example.com', 'port': 32274 }, ] Returns a list of dictionaries """ inst = fabric_cnx or self.get_instance() result = [] err_msg = "Looking up Fabric servers failed using {host}:{port}: {err}" try: (fabric_uuid_str, version, ttl, addr_list) = inst.proxy.dump.fabric_nodes() for addr in addr_list: try: host, port = addr.split(':', 2) port = int(port) except ValueError: host, port = addr, MYSQL_FABRIC_PORT result.append({'host': host, 'port': port}) except (Fault, socket.error) as exc: msg = err_msg.format(err=str(exc), host=inst.host, port=inst.port) raise InterfaceError(msg) except (TypeError, AttributeError) as exc: msg = err_msg.format( err="No Fabric server available ({0})".format(exc), host=inst.host, port=inst.port) raise InterfaceError(msg) try: fabric_uuid = uuid.UUID('{' + fabric_uuid_str + '}') except TypeError: fabric_uuid = uuid.uuid4() return fabric_uuid, version, ttl, result def get_group_servers(self, group, use_cache=True): """Get all MySQL servers in a group This method returns information about all MySQL part of the given high-availability group. When use_cache is set to True, the cached information will be used. Raises InterfaceError on errors. Returns list of FabricMySQLServer objects. """ # Get group information from cache if use_cache: entry = self._cache.group_search(group) if entry: # Cache group information return entry.servers inst = self.get_instance() result = [] try: servers = inst.proxy.dump.servers( self._version_token, group)[3] except (Fault, socket.error) as exc: msg = ("Looking up MySQL servers failed for group " "{group}: {error}").format(error=str(exc), group=group) raise InterfaceError(msg) weights = [] for server in servers: # We make sure, when using local groups, we skip the global group if server[1] == group: server[3] = int(server[3]) # port should be an int mysqlserver = FabricMySQLServer(*server) result.append(mysqlserver) if mysqlserver.status == STATUS_SECONDARY: weights.append((mysqlserver.uuid, mysqlserver.weight)) self._cache.cache_group(group, result) if weights: self._group_balancers[group] = WeightedRoundRobin(*weights) return result def get_group_server(self, group, mode=None, status=None): """Get a MySQL server from a group The method uses MySQL Fabric to get the correct MySQL server for the specified group. You can specify mode or status, but not both. The mode argument will decide whether the primary or a secondary server is returned. When no secondary server is available, the primary is returned. Status is used to force getting either a primary or a secondary. The returned tuple contains host, port and uuid. Raises InterfaceError on errors; ValueError when both mode and status are given. Returns a FabricMySQLServer object. """ if mode and status: raise ValueError( "Either mode or status must be given, not both") errmsg = "No MySQL server available for group '{group}'" servers = self.get_group_servers(group, use_cache=True) if not servers: raise InterfaceError(errmsg.format(group=group)) # Get the Master and return list (host, port, UUID) primary = None secondary = [] for server in servers: if server.status == STATUS_SECONDARY: secondary.append(server) elif server.status == STATUS_PRIMARY: primary = server if mode in (MODE_WRITEONLY, MODE_READWRITE) or status == STATUS_PRIMARY: if not primary: self.reset_cache(group=group) raise InterfaceError((errmsg + ' {query}={value}').format( query='status' if status else 'mode', group=group, value=status or mode)) return primary # Return primary if no secondary is available if not secondary and primary: return primary elif group in self._group_balancers: next_secondary = self._group_balancers[group].get_next()[0] for mysqlserver in secondary: if next_secondary == mysqlserver.uuid: return mysqlserver self.reset_cache(group=group) raise InterfaceError(errmsg.format(group=group, mode=mode)) def get_sharding_information(self, tables=None, database=None): """Get and cache the sharding information for given tables This method is fetching sharding information from MySQL Fabric and caches the result. The tables argument must be sequence of sequences contain the name of the database and table. If no database is given, the value for the database argument will be used. Examples: tables = [('salary',), ('employees',)] get_sharding_information(tables, database='employees') tables = [('salary', 'employees'), ('employees', employees)] get_sharding_information(tables) Raises InterfaceError on errors; ValueError when something is wrong with the tables argument. """ if not isinstance(tables, (list, tuple)): raise ValueError("tables should be a sequence") patterns = [] for table in tables: if not isinstance(table, (list, tuple)) and not database: raise ValueError("No database specified for table {0}".format( table)) if isinstance(table, (list, tuple)): dbase = table[1] tbl = table[0] else: dbase = database tbl = table patterns.append("{0}.{1}".format(dbase, tbl)) inst = self.get_instance() try: result = inst.proxy.dump.sharding_information( self._version_token, ','.join(patterns)) except (Fault, socket.error) as exc: msg = "Looking up sharding information failed : {error}".format( error=str(exc)) raise InterfaceError(msg) for info in result[3]: self._cache.sharding_cache_table(FabricShard(*info)) def get_shard_server(self, tables, key, scope=SCOPE_LOCAL, mode=None): """Get MySQL server information for a particular shard Raises DatabaseError when the table is unknown or when tables are not on the same shard. ValueError is raised when there is a problem with the methods arguments. InterfaceError is raised for other errors. """ if not isinstance(tables, (list, tuple)): raise ValueError("tables should be a sequence") groups = [] for dbobj in tables: try: database, table = dbobj.split('.') except ValueError: raise ValueError( "tables should be given as <database>.<table>, " "was {0}".format(dbobj)) entry = self._cache.sharding_search(database, table) if not entry: self.get_sharding_information((table,), database) entry = self._cache.sharding_search(database, table) if not entry: raise DatabaseError( errno=errorcode.ER_BAD_TABLE_ERROR, msg="Unknown table '{database}.{table}'".format( database=database, table=table)) if scope == 'GLOBAL': return self.get_group_server(entry.global_group, mode=mode) if entry.shard_type == 'RANGE': partitions = sorted(entry.partitioning.keys()) index = partitions[bisect(partitions, int(key)) - 1] partition = entry.partitioning[index] elif entry.shard_type == 'HASH': md5key = md5(str(key)) partition_keys = sorted( entry.partitioning.keys(), reverse=True) index = partition_keys[-1] for partkey in partition_keys: if md5key.digest() >= b16decode(partkey): index = partkey break partition = entry.partitioning[index] else: raise InterfaceError( "Unsupported sharding type {0}".format(entry.shard_type)) groups.append(partition['group']) if not all(group == groups[0] for group in groups): raise DatabaseError( "Tables are located in different shards.") return self.get_group_server(groups[0], mode=mode)
class Fabric(object): """Class managing MySQL Fabric instances""" def __init__(self, host, username=None, password=None, port=MYSQL_FABRIC_PORT, connect_attempts=_CNX_ATTEMPT_MAX, connect_delay=_CNX_ATTEMPT_DELAY, report_errors=False, ssl_ca=None, ssl_key=None, ssl_cert=None, user=None): """Initialize""" self._fabric_instances = {} self._fabric_uuid = None self._ttl = 1 * 60 # one minute by default self._version_token = None self._connect_attempts = connect_attempts self._connect_delay = connect_delay self._cache = FabricCache() self._group_balancers = {} self._init_host = host self._init_port = port self._ssl = _validate_ssl_args(ssl_ca, ssl_key, ssl_cert) self._report_errors = report_errors if user and username: raise ValueError("can not specify both user and username") self._username = user or username self._password = password @property def username(self): return self._username @property def password(self): return self._password @property def ssl_config(self): return self._ssl def seed(self, host=None, port=None): """Get MySQL Fabric Instances This method uses host and port to connect to a MySQL Fabric server and get all the instances managing the same metadata. Raises InterfaceError on errors. """ host = host or self._init_host port = port or self._init_port fabinst = FabricConnection(self, host, port, connect_attempts=self._connect_attempts, connect_delay=self._connect_delay) fabinst.connect() fabric_uuid, version, ttl, fabrics = self.get_fabric_servers(fabinst) if not fabrics: # Raise, something went wrong. raise InterfaceError("Failed getting list of Fabric servers") if self._version_token == version: return _LOGGER.info("Loading Fabric configuration version {version}".format( version=version)) self._fabric_uuid = fabric_uuid self._version_token = version if ttl > 0: self._ttl = ttl # Update the Fabric servers for fabric in fabrics: inst = FabricConnection(self, fabric['host'], fabric['port'], connect_attempts=self._connect_attempts, connect_delay=self._connect_delay) inst_uuid = inst.uuid if inst_uuid not in self._fabric_instances: inst.connect() self._fabric_instances[inst_uuid] = inst _LOGGER.debug("Added new Fabric server {host}:{port}".format( host=inst.host, port=inst.port)) def reset_cache(self, group=None): """Reset cached information This method destroys all cached information. """ if group: _LOGGER.debug( "Resetting cache for group '{group}'".format(group=group)) self.get_group_servers(group, use_cache=False) else: _LOGGER.debug("Resetting cache") self._cache = FabricCache() def get_instance(self): """Get a MySQL Fabric Instance This method will get the next available MySQL Fabric Instance. Raises InterfaceError when no instance is available or connected. """ nxt = 0 errmsg = "No MySQL Fabric instance available" if not self._fabric_instances: raise InterfaceError(errmsg + " (not seeded?)") if sys.version_info[0] == 2: instance_list = self._fabric_instances.keys() inst = self._fabric_instances[instance_list[nxt]] else: inst = self._fabric_instances[list(self._fabric_instances)[nxt]] if not inst.is_connected: inst.connect() return inst def report_failure(self, server_uuid, errno): """Report failure to Fabric This method sets the status of a MySQL server identified by server_uuid. """ if not self._report_errors: return errno = int(errno) current_host = socket.getfqdn() if errno in REPORT_ERRORS or errno in REPORT_ERRORS_EXTRA: _LOGGER.debug("Reporting error %d of server %s", errno, server_uuid) inst = self.get_instance() try: inst.proxy.threat.report_failure(server_uuid, current_host, errno) except (Fault, socket.error) as exc: _LOGGER.debug("Failed reporting server to Fabric.") # Not requiring further action def get_fabric_servers(self, fabric_cnx=None): """Get all MySQL Fabric instances This method looks up the other MySQL Fabric instances which uses the same metadata. The returned list contains dictionaries with connection information such ass host and port. For example: [ {'host': 'fabric_prod_1.example.com', 'port': 32274 }, {'host': 'fabric_prod_2.example.com', 'port': 32274 }, ] Returns a list of dictionaries """ inst = fabric_cnx or self.get_instance() result = [] err_msg = "Looking up Fabric servers failed using {host}:{port}: {err}" try: (fabric_uuid_str, version, ttl, addr_list) = inst.proxy.dump.fabric_nodes() for addr in addr_list: try: host, port = addr.split(':', 2) port = int(port) except ValueError: host, port = addr, MYSQL_FABRIC_PORT result.append({'host': host, 'port': port}) except (Fault, socket.error) as exc: msg = err_msg.format(err=str(exc), host=inst.host, port=inst.port) raise InterfaceError(msg) except (TypeError, AttributeError) as exc: msg = err_msg.format( err="No Fabric server available ({0})".format(exc), host=inst.host, port=inst.port) raise InterfaceError(msg) try: fabric_uuid = uuid.UUID('{' + fabric_uuid_str + '}') except TypeError: fabric_uuid = uuid.uuid4() return fabric_uuid, version, ttl, result def get_group_servers(self, group, use_cache=True): """Get all MySQL servers in a group This method returns information about all MySQL part of the given high-availability group. When use_cache is set to True, the cached information will be used. Raises InterfaceError on errors. Returns list of FabricMySQLServer objects. """ # Get group information from cache if use_cache: entry = self._cache.group_search(group) if entry: # Cache group information return entry.servers inst = self.get_instance() result = [] try: servers = inst.proxy.dump.servers(self._version_token, group)[3] except (Fault, socket.error) as exc: msg = ("Looking up MySQL servers failed for group " "{group}: {error}").format(error=str(exc), group=group) raise InterfaceError(msg) weights = [] for server in servers: # We make sure, when using local groups, we skip the global group if server[1] == group: server[3] = int(server[3]) # port should be an int mysqlserver = FabricMySQLServer(*server) result.append(mysqlserver) if mysqlserver.status == STATUS_SECONDARY: weights.append((mysqlserver.uuid, mysqlserver.weight)) self._cache.cache_group(group, result) if weights: self._group_balancers[group] = WeightedRoundRobin(*weights) return result def get_group_server(self, group, mode=None, status=None): """Get a MySQL server from a group The method uses MySQL Fabric to get the correct MySQL server for the specified group. You can specify mode or status, but not both. The mode argument will decide whether the primary or a secondary server is returned. When no secondary server is available, the primary is returned. Status is used to force getting either a primary or a secondary. The returned tuple contains host, port and uuid. Raises InterfaceError on errors; ValueError when both mode and status are given. Returns a FabricMySQLServer object. """ if mode and status: raise ValueError("Either mode or status must be given, not both") errmsg = "No MySQL server available for group '{group}'" servers = self.get_group_servers(group, use_cache=True) if not servers: raise InterfaceError(errmsg.format(group=group)) # Get the Master and return list (host, port, UUID) primary = None secondary = [] for server in servers: if server.status == STATUS_SECONDARY: secondary.append(server) elif server.status == STATUS_PRIMARY: primary = server if mode in (MODE_WRITEONLY, MODE_READWRITE) or status == STATUS_PRIMARY: if not primary: self.reset_cache(group=group) raise InterfaceError((errmsg + ' {query}={value}').format( query='status' if status else 'mode', group=group, value=status or mode)) return primary # Return primary if no secondary is available if not secondary and primary: return primary elif group in self._group_balancers: next_secondary = self._group_balancers[group].get_next()[0] for mysqlserver in secondary: if next_secondary == mysqlserver.uuid: return mysqlserver self.reset_cache(group=group) raise InterfaceError(errmsg.format(group=group, mode=mode)) def get_sharding_information(self, tables=None, database=None): """Get and cache the sharding information for given tables This method is fetching sharding information from MySQL Fabric and caches the result. The tables argument must be sequence of sequences contain the name of the database and table. If no database is given, the value for the database argument will be used. Examples: tables = [('salary',), ('employees',)] get_sharding_information(tables, database='employees') tables = [('salary', 'employees'), ('employees', employees)] get_sharding_information(tables) Raises InterfaceError on errors; ValueError when something is wrong with the tables argument. """ if not isinstance(tables, (list, tuple)): raise ValueError("tables should be a sequence") patterns = [] for table in tables: if not isinstance(table, (list, tuple)) and not database: raise ValueError( "No database specified for table {0}".format(table)) if isinstance(table, (list, tuple)): dbase = table[1] tbl = table[0] else: dbase = database tbl = table patterns.append("{0}.{1}".format(dbase, tbl)) inst = self.get_instance() try: result = inst.proxy.dump.sharding_information( self._version_token, ','.join(patterns)) except (Fault, socket.error) as exc: msg = "Looking up sharding information failed : {error}".format( error=str(exc)) raise InterfaceError(msg) for info in result[3]: self._cache.sharding_cache_table(FabricShard(*info)) def get_shard_server(self, tables, key, scope=SCOPE_LOCAL, mode=None): """Get MySQL server information for a particular shard Raises DatabaseError when the table is unknown or when tables are not on the same shard. ValueError is raised when there is a problem with the methods arguments. InterfaceError is raised for other errors. """ if not isinstance(tables, (list, tuple)): raise ValueError("tables should be a sequence") groups = [] for dbobj in tables: try: database, table = dbobj.split('.') except ValueError: raise ValueError( "tables should be given as <database>.<table>, " "was {0}".format(dbobj)) entry = self._cache.sharding_search(database, table) if not entry: self.get_sharding_information((table, ), database) entry = self._cache.sharding_search(database, table) if not entry: raise DatabaseError( errno=errorcode.ER_BAD_TABLE_ERROR, msg="Unknown table '{database}.{table}'".format( database=database, table=table)) if scope == 'GLOBAL': return self.get_group_server(entry.global_group, mode=mode) if entry.shard_type == 'RANGE': partitions = sorted(entry.partitioning.keys()) index = partitions[bisect(partitions, int(key)) - 1] partition = entry.partitioning[index] elif entry.shard_type == 'HASH': md5key = md5(str(key)) partition_keys = sorted(entry.partitioning.keys(), reverse=True) index = partition_keys[-1] for partkey in partition_keys: if md5key.digest() >= b16decode(partkey): index = partkey break partition = entry.partitioning[index] else: raise InterfaceError("Unsupported sharding type {0}".format( entry.shard_type)) groups.append(partition['group']) if not all(group == groups[0] for group in groups): raise DatabaseError("Tables are located in different shards.") return self.get_group_server(groups[0], mode=mode)