def get_group_server(self, group, mode=None, status=None): """Get a MySQL server from a group The method uses MySQL Fabric to get the correct MySQL server for the specified group. You can specify mode or status, but not both. The mode argument will decide whether the primary or a secondary server is returned. When no secondary server is available, the primary is returned. Status is used to force getting either a primary or a secondary. The returned tuple contains host, port and uuid. Raises InterfaceError on errors; ValueError when both mode and status are given. Returns a FabricMySQLServer object. """ if mode and status: raise ValueError("Either mode or status must be given, not both") errmsg = "No MySQL server available for group '{group}'" servers = self.get_group_servers(group, use_cache=True) if not servers: raise InterfaceError(errmsg.format(group=group)) # Get the Master and return list (host, port, UUID) primary = None secondary = [] for server in servers: if server.status == STATUS_SECONDARY: secondary.append(server) elif server.status == STATUS_PRIMARY: primary = server if mode in (MODE_WRITEONLY, MODE_READWRITE) or status == STATUS_PRIMARY: if not primary: self.reset_cache(group=group) raise InterfaceError((errmsg + ' {query}={value}').format( query='status' if status else 'mode', group=group, value=status or mode)) return primary # Return primary if no secondary is available if not secondary and primary: return primary elif group in self._group_balancers: next_secondary = self._group_balancers[group].get_next()[0] for mysqlserver in secondary: if next_secondary == mysqlserver.uuid: return mysqlserver self.reset_cache(group=group) raise InterfaceError(errmsg.format(group=group, mode=mode))
def _xmlrpc_get_proxy(self): """Return the XMLRPC server proxy instance to MySQL Fabric This method tries to get a valid connection to a MySQL Fabric server. Returns a XMLRPC ServerProxy instance. """ if self.is_connected: return self._proxy attempts = self._connect_attempts delay = self._connect_delay proxy = None counter = 0 while counter != attempts: counter += 1 try: proxy = ServerProxy(self.uri) proxy._some_nonexisting_method() except Fault: # We are actually connected return proxy except socket.error as exc: if counter == attempts: raise InterfaceError( "Connection to MySQL Fabric failed ({0})".format(exc)) _LOGGER.debug( "Retrying {host}:{port}, attempts {counter}".format( host=self.host, port=self.port, counter=counter)) if delay > 0: time.sleep(delay)
def connect(*args, **kwargs): """Create or get a MySQL connection object In its simpliest form, Connect() will open a connection to a MySQL server and return a MySQLConnection object. When any connection pooling arguments are given, for example pool_name or pool_size, a pool is created or a previously one is used to return a PooledMySQLConnection. Returns MySQLConnection or PooledMySQLConnection. """ if all(['fabric' in kwargs, 'failover' in kwargs]): raise InterfaceError("fabric and failover arguments can not be used") if 'fabric' in kwargs: return mysql.connector.fabric.connect(*args, **kwargs) # Failover if 'failover' in kwargs: return _get_failover_connection(**kwargs) # Pooled connections if any([key in kwargs for key in CNX_POOL_ARGS]): return _get_pooled_connection(**kwargs) # Regular connection return MySQLConnection(*args, **kwargs)
def _get_pooled_connection(**kwargs): """Return a pooled MySQL connection""" # If no pool name specified, generate one from .pooling import (MySQLConPooL, generate_pool_name, CONNECTION_POOL_LOCK) try: pool_name = kwargs['pool_name'] except KeyError: pool_name = generate_pool_name(**kwargs) # Setup the pool, ensuring only 1 thread can update at a time with CONNECTION_POOL_LOCK: if pool_name not in _CONNECTION_POOLS: _CONNECTION_POOLS[pool_name] = MySQLConPool(**kwargs) elif isinstance(_CONNECTION_POOLS[pool_name], MySQLConPool): # pool_size must be the same check_size = _CONNECTION_POOLS[pool_name].pool_size if ('pool_size' in kwargs and kwargs['pool_size'] != check_size): raise PoolError("Size can not be changed " "for active pools.") # Return pooled connection try: return _CONNECTION_POOLS[pool_name].get_connection() except AttributeError: raise InterfaceError( "Failed getting connection from pool '{0}'".format(pool_name))
def get_shard_server(self, tables, key, scope=SCOPE_LOCAL, mode=None): """Get MySQL server information for a particular shard Raises DatabaseError when the table is unknown or when tables are not on the same shard. ValueError is raised when there is a problem with the methods arguments. InterfaceError is raised for other errors. """ if not isinstance(tables, (list, tuple)): raise ValueError("tables should be a sequence") groups = [] for dbobj in tables: try: database, table = dbobj.split('.') except ValueError: raise ValueError( "tables should be given as <database>.<table>, " "was {0}".format(dbobj)) entry = self._cache.sharding_search(database, table) if not entry: self.get_sharding_information((table,), database) entry = self._cache.sharding_search(database, table) if not entry: raise DatabaseError( errno=errorcode.ER_BAD_TABLE_ERROR, msg="Unknown table '{database}.{table}'".format( database=database, table=table)) if scope == 'GLOBAL': return self.get_group_server(entry.global_group, mode=mode) if entry.shard_type == 'RANGE': partitions = sorted(entry.partitioning.keys()) index = partitions[bisect(partitions, int(key)) - 1] partition = entry.partitioning[index] elif entry.shard_type == 'HASH': md5key = md5(str(key)) partition_keys = sorted( entry.partitioning.keys(), reverse=True) index = partition_keys[-1] for partkey in partition_keys: if md5key.digest() >= b16decode(partkey): index = partkey break partition = entry.partitioning[index] else: raise InterfaceError( "Unsupported sharding type {0}".format(entry.shard_type)) groups.append(partition['group']) if not all(group == groups[0] for group in groups): raise DatabaseError( "Tables are located in different shards.") return self.get_group_server(groups[0], mode=mode)
def get_fabric_servers(self, fabric_cnx=None): """Get all MySQL Fabric instances This method looks up the other MySQL Fabric instances which uses the same metadata. The returned list contains dictionaries with connection information such ass host and port. For example: [ {'host': 'fabric_prod_1.example.com', 'port': 32274 }, {'host': 'fabric_prod_2.example.com', 'port': 32274 }, ] Returns a list of dictionaries """ inst = fabric_cnx or self.get_instance() result = [] err_msg = "Looking up Fabric servers failed using {host}:{port}: {err}" try: (fabric_uuid_str, version, ttl, addr_list) = inst.proxy.dump.fabric_nodes() for addr in addr_list: try: host, port = addr.split(':', 2) port = int(port) except ValueError: host, port = addr, MYSQL_FABRIC_PORT result.append({'host': host, 'port': port}) except (Fault, socket.error) as exc: msg = err_msg.format(err=str(exc), host=inst.host, port=inst.port) raise InterfaceError(msg) except (TypeError, AttributeError) as exc: msg = err_msg.format( err="No Fabric server available ({0})".format(exc), host=inst.host, port=inst.port) raise InterfaceError(msg) try: fabric_uuid = uuid.UUID('{' + fabric_uuid_str + '}') except TypeError: fabric_uuid = uuid.uuid4() return fabric_uuid, version, ttl, result
def request(self, host, handler, request_body, verbose): uri = '{scheme}://{host}{handler}'.format(scheme=self._scheme, host=host, handler=handler) if self._passmgr: self._passmgr.add_password(None, uri, self._username, self._password) if self.verbose: _LOGGER.debug("FabricTransport: {0}".format(uri)) opener = urllib2.build_opener(*self._handlers) headers = { 'Content-Type': 'text/xml', 'User-Agent': self.user_agent, } req = urllib2.Request(uri, request_body, headers=headers) try: return self.parse_response(opener.open(req)) except urllib2.URLError as exc: try: if exc.code == 400: reason = 'Permission denied' else: reason = exc.reason msg = "{reason} ({code})".format(reason=reason, code=exc.code) except AttributeError: if 'SSL' in str(exc): msg = "SSL error" else: msg = str(exc) raise InterfaceError("Connection with Fabric failed: " + msg) except BadStatusLine: raise InterfaceError("Connection with Fabric failed: check SSL")
def get_sharding_information(self, tables=None, database=None): """Get and cache the sharding information for given tables This method is fetching sharding information from MySQL Fabric and caches the result. The tables argument must be sequence of sequences contain the name of the database and table. If no database is given, the value for the database argument will be used. Examples: tables = [('salary',), ('employees',)] get_sharding_information(tables, database='employees') tables = [('salary', 'employees'), ('employees', employees)] get_sharding_information(tables) Raises InterfaceError on errors; ValueError when something is wrong with the tables argument. """ if not isinstance(tables, (list, tuple)): raise ValueError("tables should be a sequence") patterns = [] for table in tables: if not isinstance(table, (list, tuple)) and not database: raise ValueError( "No database specified for table {0}".format(table)) if isinstance(table, (list, tuple)): dbase = table[1] tbl = table[0] else: dbase = database tbl = table patterns.append("{0}.{1}".format(dbase, tbl)) inst = self.get_instance() try: result = inst.proxy.dump.sharding_information( self._version_token, ','.join(patterns)) except (Fault, socket.error) as exc: msg = "Looking up sharding information failed : {error}".format( error=str(exc)) raise InterfaceError(msg) for info in result[3]: self._cache.sharding_cache_table(FabricShard(*info))
def seed(self, host=None, port=None): """Get MySQL Fabric Instances This method uses host and port to connect to a MySQL Fabric server and get all the instances managing the same metadata. Raises InterfaceError on errors. """ host = host or self._init_host port = port or self._init_port fabinst = FabricConnection(self, host, port, connect_attempts=self._connect_attempts, connect_delay=self._connect_delay) fabinst.connect() fabric_uuid, version, ttl, fabrics = self.get_fabric_servers(fabinst) if not fabrics: # Raise, something went wrong. raise InterfaceError("Failed getting list of Fabric servers") if self._version_token == version: return _LOGGER.info("Loading Fabric configuration version {version}".format( version=version)) self._fabric_uuid = fabric_uuid self._version_token = version if ttl > 0: self._ttl = ttl # Update the Fabric servers for fabric in fabrics: inst = FabricConnection(self, fabric['host'], fabric['port'], connect_attempts=self._connect_attempts, connect_delay=self._connect_delay) inst_uuid = inst.uuid if inst_uuid not in self._fabric_instances: inst.connect() self._fabric_instances[inst_uuid] = inst _LOGGER.debug("Added new Fabric server {host}:{port}".format( host=inst.host, port=inst.port))
def connect(*args, **kwargs): """Create or get a MySQL connection object In its simpliest form, Connect() will open a connection to a MySQL server and return a MySQLConnection object. When any connection pooling arguments are given, for example pool_name or pool_size, a pool is created or a previously one is used to return a PooledMySQLConnection. Returns MySQLConnection or PooledMySQLConnection. """ if 'fabric' in kwargs: return mysql.connector.fabric.connect(*args, **kwargs) # Pooled connections if any([key in kwargs for key in CNX_POOL_ARGS]): # If no pool name specified, generate one try: pool_name = kwargs['pool_name'] except KeyError: pool_name = generate_pool_name(**kwargs) # Setup the pool, ensuring only 1 thread can update at a time with CONNECTION_POOL_LOCK: if pool_name not in _CONNECTION_POOLS: _CONNECTION_POOLS[pool_name] = MySQLConnectionPool( *args, **kwargs) elif isinstance(_CONNECTION_POOLS[pool_name], MySQLConnectionPool): # pool_size must be the same check_size = _CONNECTION_POOLS[pool_name].pool_size if ('pool_size' in kwargs and kwargs['pool_size'] != check_size): raise PoolError("Size can not be changed " "for active pools.") # Return pooled connection try: return _CONNECTION_POOLS[pool_name].get_connection() except AttributeError: raise InterfaceError( "Failed getting connection from pool '{0}'".format(pool_name)) # Regular connection return MySQLConnection(*args, **kwargs)
def fetchall(self): """Returns all rows of a query result set """ if not self._have_unread_result(): from mysql.connector.errors import InterfaceError raise InterfaceError(self.ERR_NO_RESULT_TO_FETCH) (rows, eof) = self._connection.get_rows() if self._nextrow[0]: rows.insert(0, self._nextrow[0]) res = [] for row in rows: res.append(self._row_to_python(row, self.description)) self._handle_eof(eof) rowcount = len(rows) if rowcount >= 0 and self._rowcount == -1: self._rowcount = 0 self._rowcount += rowcount return res
def get_group_servers(self, group, use_cache=True): """Get all MySQL servers in a group This method returns information about all MySQL part of the given high-availability group. When use_cache is set to True, the cached information will be used. Raises InterfaceError on errors. Returns list of FabricMySQLServer objects. """ # Get group information from cache if use_cache: entry = self._cache.group_search(group) if entry: # Cache group information _LOGGER.debug("Using cached group information") return entry.servers inst = self.get_instance() result = [] try: servers = inst.proxy.store.dump_servers(self._version_token, group)[3] except (Fault, socket.error) as exc: msg = ("Looking up MySQL servers failed for group " "{group}: {error}").format(error=str(exc), group=group) raise InterfaceError(msg) weights = [] for server in servers: # We make sure, when using local groups, we skip the global group if server[1] == group: server[3] = int(server[3]) # port should be an int mysqlserver = FabricMySQLServer(*server) result.append(mysqlserver) if mysqlserver.status == STATUS_SECONDARY: weights.append((mysqlserver.uuid, mysqlserver.weight)) self._cache.cache_group(group, result) if weights: self._group_balancers[group] = WeightedRoundRobin(*weights) return result
def _xmlrpc_get_proxy(self): """Return the XMLRPC server proxy instance to MySQL Fabric This method tries to get a valid connection to a MySQL Fabric server. Returns a XMLRPC ServerProxy instance. """ if self.is_connected: return self._proxy attempts = self._connect_attempts delay = self._connect_delay proxy = None counter = 0 while counter != attempts: counter += 1 try: if self._fabric.ssl_config: scheme = 'https' https_handler = FabricHTTPSHandler(self._fabric.ssl_config) else: https_handler = None scheme = 'http' transport = FabricTransport(self._fabric.username, self._fabric.password, verbose=0, https_handler=https_handler) proxy = ServerProxy(self.uri, transport=transport, verbose=0) proxy._some_nonexisting_method() # pylint: disable=W0212 except Fault: # We are actually connected return proxy except socket.error as exc: if counter == attempts: raise InterfaceError( "Connection to MySQL Fabric failed ({0})".format(exc)) _LOGGER.debug( "Retrying {host}:{port}, attempts {counter}".format( host=self.host, port=self.port, counter=counter)) if delay > 0: time.sleep(delay)
def get_instance(self): """Get a MySQL Fabric Instance This method will get the next available MySQL Fabric Instance. Raises InterfaceError when no instance is available or connected. """ nxt = 0 errmsg = "No MySQL Fabric instance available" if not self._fabric_instances: raise InterfaceError(errmsg + " (not seeded?)") if sys.version_info[0] == 2: instance_list = self._fabric_instances.keys() inst = self._fabric_instances[instance_list[nxt]] else: inst = self._fabric_instances[list(self._fabric_instances)[nxt]] if not inst.is_connected: inst.connect() return inst
def _get_failover_connection(**kwargs): """Return a MySQL connection and try to failover if needed An InterfaceError is raise when no MySQL is available. ValueError is raised when the failover server configuration contains an illegal connection argument. Supported arguments are user, password, host, port, unix_socket and database. ValueError is also raised when the failover argument was not provided. Returns MySQLConnection instance. """ config = kwargs.copy() try: failover = config['failover'] except KeyError: raise ValueError('failover argument not provided') del config['failover'] support_cnx_args = set([ 'user', 'password', 'host', 'port', 'unix_socket', 'database', 'pool_name', 'pool_size' ]) # First check if we can add all use the configuration for server in failover: diff = set(server.keys()) - support_cnx_args if diff: raise ValueError( "Unsupported connection argument {0} in failover: {1}".format( 's' if len(diff) > 1 else '', ', '.join(diff))) for server in failover: new_config = config.copy() new_config.update(server) try: return connect(**new_config) except Error: # If we failed to connect, we try the next server pass raise InterfaceError("Could not failover: no MySQL server available")
def connect(*args, **kwargs): """Create or get a MySQL connection object In its simpliest form, Connect() will open a connection to a MySQL server and return a AioSQLConnection object. When any connection pooling arguments are given, for example pool_name or pool_size, a pool is created or a previously one is used to return a PooledAioSQLConnection. Returns AioSQLConnection or PooledAioSQLConnection. """ # Option files if 'option_files' in kwargs: new_config = read_option_files(**kwargs) return connect(**new_config) if all(['fabric' in kwargs, 'failover' in kwargs]): raise InterfaceError("fabric and failover arguments can not be used") if 'fabric' in kwargs: from .fabric import connect as fabric_connect return fabric_connect(*args, **kwargs) # Failover if 'failover' in kwargs: return _get_failover_connection(**kwargs) # Pooled connections try: from mysql.connector.pooling import CNX_POOL_ARGS if any([key in kwargs for key in CNX_POOL_ARGS]): return _get_pooled_connection(**kwargs) except NameError: # No pooling pass # Regular connection return AioMySQLConnection(*args, **kwargs)
def read_packet(self): """Read a MySQL packet from the socket This method reads a MySQL packet form the socket, parses it and returns the type and the payload. The type is the value of the first byte of the payload. :return: Tuple with type and payload of packet. :rtype: tuple """ header = bytearray(b'') header_len = 0 while header_len < 4: chunk = self.request.recv(4 - header_len) if not chunk: raise InterfaceError(errno=2013) header += chunk header_len = len(header) length = struct.unpack_from('<I', buffer(header[0:3] + b'\x00'))[0] self._curr_pktnr = header[-1] data = bytearray(self.request.recv(length)) return data[0], data[1:]
def _connect(self): """Get a MySQL server based on properties and connect This method gets a MySQL server from MySQL Fabric using already properties set using the set_property() method. You can specify how many times and the delay between trying using attempts and attempt_delay. Raises InterfaceError on errors. A ValueError is raised when both group and sharding are specified, or neither of them. """ if self.is_connected(): return props = self._cnx_properties attempts = props['attempts'] attempt_delay = props['attempt_delay'] if props['group'] and props['tables']: raise ValueError( "Either 'group' or 'tables' property can be set, not both.") if not (props['group'] or props['tables']): raise ValueError( "Either 'group' or 'tables' property needs to be set.") dbconfig = self._mysql_config.copy() counter = 0 while counter != attempts: counter += 1 try: group = None if props['tables']: if props['scope'] == 'LOCAL' and not props['key']: raise ValueError( "Scope 'LOCAL' needs key property to be set") mysqlserver = self._fabric.get_shard_server( props['tables'], props['key'], scope=props['scope'], mode=props['mode']) elif props['group']: group = props['group'] mysqlserver = self._fabric.get_group_server( group, mode=props['mode']) else: raise InterfaceError( "Missing group or key and tables properties") except InterfaceError as err: _LOGGER.debug( "Trying to get MySQL server (attempt {0}; {1})".format( counter, err)) if counter == attempts: raise InterfaceError( "Error getting connection: {0}".format(err)) if attempt_delay > 0: time.sleep(attempt_delay) continue # Make sure we do not change the stored configuration dbconfig['host'] = mysqlserver.host dbconfig['port'] = mysqlserver.port try: self._mysql_cnx = mysql.connector.connect(**dbconfig) except InterfaceError as err: self._fabric.set_server_status(mysqlserver.uuid, STATUS_FAULTY) self.reset_cache(mysqlserver.group) if counter == attempts: raise InterfaceError( "Reported faulty server to Fabric ({0})".format(err)) else: self._fabric_mysql_server = mysqlserver break
def _connect(self): """Get a MySQL server based on properties and connect This method gets a MySQL server from MySQL Fabric using already properties set using the set_property() method. You can specify how many times and the delay between trying using attempts and attempt_delay. Raises ValueError when there are problems with arguments or properties; InterfaceError on connectivity errors. """ if self.is_connected(): return props = self._cnx_properties attempts = props['attempts'] attempt_delay = props['attempt_delay'] dbconfig = self._mysql_config.copy() counter = 0 while counter != attempts: counter += 1 try: group = None if props['tables']: if props['scope'] == 'LOCAL' and not props['key']: raise ValueError( "Scope 'LOCAL' needs key property to be set") mysqlserver = self._fabric.get_shard_server( props['tables'], props['key'], scope=props['scope'], mode=props['mode']) elif props['group']: group = props['group'] mysqlserver = self._fabric.get_group_server( group, mode=props['mode']) else: raise ValueError( "Missing group or key and tables properties") except InterfaceError as exc: _LOGGER.debug( "Trying to get MySQL server (attempt {0}; {1})".format( counter, exc)) if counter == attempts: raise InterfaceError( "Error getting connection: {0}".format(exc)) if attempt_delay > 0: _LOGGER.debug("Waiting {0}".format(attempt_delay)) time.sleep(attempt_delay) continue # Make sure we do not change the stored configuration dbconfig['host'] = mysqlserver.host dbconfig['port'] = mysqlserver.port try: self._mysql_cnx = mysql.connector.connect(**dbconfig) except Error as exc: if counter == attempts: self.reset_cache(mysqlserver.group) self._fabric.report_failure(mysqlserver.uuid, exc.errno) raise InterfaceError( "Reported faulty server to Fabric ({0})".format(exc)) if attempt_delay > 0: time.sleep(attempt_delay) continue else: self._fabric_mysql_server = mysqlserver break