def setup(self): """Setup the MySQLRPC request handler""" self._handshaked = False self._authenticated = False self._curr_pktnr = 1 persistence.init_thread() self._store = persistence.current_persister()
def check_number_threads(increasing=0): """Check the number of threads that are running and whether the maximum number of connections in the state store is configured accordingly. :param increasing: Whether you want to increase the number of threads and how many threads. Default is zero. It raises a ConfigurationError exception if the number of connections is too small. """ from mysql.fabric import ( errors as _errors, executor as _executor, persistence as _persistence, services as _services, server as _server, ) n_sessions = _services.ServiceManager().get_number_sessions() n_executors = _executor.Executor().get_number_executors() n_failure_detectors = \ len(_server.Group.groups_by_status(_server.Group.ACTIVE)) n_controls = 1 persister = _persistence.current_persister() max_allowed_connections = persister.max_allowed_connections() if (n_sessions + n_executors + n_controls + n_failure_detectors +\ increasing) > (max_allowed_connections - 1): raise _errors.ConfigurationError( "Too many threads requested. Session threads (%s), Executor " "threads (%s), Control threads (%s) and Failure Detector threads " "(%s). The maximum number of threads allowed is (%s). Increase " "the maximum number of connections in the state store in order " "to increase this limit." % (n_sessions, n_executors, n_controls, n_failure_detectors, max_allowed_connections - 1) )
def check_number_threads(increasing=0): """Check the number of threads that are running and whether the maximum number of connections in the state store is configured accordingly. :param increasing: Whether you want to increase the number of threads and how many threads. Default is zero. It raises a ConfigurationError exception if the number of connections is too small. """ from mysql.fabric import ( errors as _errors, executor as _executor, persistence as _persistence, services as _services, server as _server, ) n_sessions = _services.ServiceManager().get_number_sessions() n_executors = _executor.Executor().get_number_executors() n_failure_detectors = \ len(_server.Group.groups_by_status(_server.Group.ACTIVE)) n_controls = 1 persister = _persistence.current_persister() max_allowed_connections = persister.max_allowed_connections() if (n_sessions + n_executors + n_controls + n_failure_detectors +\ increasing) > (max_allowed_connections - 1): raise _errors.ConfigurationError( "Too many threads requested. Session threads (%s), Executor " "threads (%s), Control threads (%s) and Failure Detector threads " "(%s). The maximum number of threads allowed is (%s). Increase " "the maximum number of connections in the state store in order " "to increase this limit." % (n_sessions, n_executors, n_controls, n_failure_detectors, max_allowed_connections - 1))
def get_user(username, protocol, persister=None): """Get user information from data store :param username: User to query for. :param protocol: Protocol used to connect. :param persister: A valid handle to the state store. :return: namedtuple with password, username, protocol and user_id :rtype: namedtuple """ if not persister: persister = _persistence.current_persister() options = { "fetch": False, "params": (username, protocol), "columns": True, } cur = persister.exec_stmt( "SELECT password, username, protocol, user_id " "FROM users WHERE username = %s " "AND protocol = %s", options) row = cur.fetchone() if row: return row return None
def dispatch(self, *args): """Setup Fabric Storage System. """ # Configure connections. _configure_connections(self.config) _persistence.init_thread() self.persister = _persistence.current_persister() self.execute(*args)
def do_POST(self): if self._server.auth_disabled: return SimpleXMLRPCRequestHandler.do_POST(self) hashfunc = self._server.hashfunc nonce = None if self.headers.has_key('authorization'): attrs = _parse_digest_header(self.headers['authorization']) if not attrs: _LOGGER.error( "Illegal digest authentication from {host}".format( host=self._client_address)) self.send_response(400) return nonce = attrs['nonce'] store = _persistence.current_persister() user = credentials.User.fetch_user(attrs['username'], protocol='xmlrpc') if not user: _LOGGER.error("Authentication failed for {user}@{host}".format( user=attrs['username'], host=self._client_address[0])) self.send_response(400) return ha1 = user.password_hash ha2 = hashfunc('POST' + ':' + self.rpc_paths[1]).hexdigest() if 'qop' in attrs and attrs['qop'] in ('auth', 'auth-int'): data = "{nonce}:{nc}:{cnonce}:{qop}:{ha2}".format( nonce=attrs['nonce'], nc=attrs['nc'], cnonce=attrs['cnonce'], qop=attrs['qop'], ha2=ha2) reqdigest = hashfunc(ha1 + ':' + data).hexdigest() else: kd = hashfunc(ha1 + ':' + nonce + ':' + ha2).hexdigest() if reqdigest == attrs['response']: self._authenticated = self._server.authorize_client( nonce, self._client_address, attrs['username'], attrs['realm'], nc=int(attrs['nc'], 16)) self._user = user else: self.send_response(400) return if not self._authenticated: nonce = _digest_nonce(self._server.hashfunc, self._client_address) self.send_authentication(nonce) return return SimpleXMLRPCRequestHandler.do_POST(self)
def _role_selection(message=None, choices=None, persister=None): """Offers user to select roles on the console :param persister: A valid handle to the state store. :param message: Message shown just before prompt. :param choices: Do not ask, just process choices (string or sequence). :return: List of role IDs or role names. :rtype: list :raises errors.CredentialError: if invalid role was given """ if not persister: persister = _persistence.current_persister() if not choices: if not message: message = "\nEnter comma separated list of role IDs or names: " choices = raw_input(message) if not choices.strip(): return [] if isinstance(choices, str): choices = choices.split(',') options = { "raw": False, "fetch": False, "columns": True, } cur = persister.exec_stmt("SELECT role_id, name FROM roles", options) valid_role_ids = [] roles = {} for row in cur: roles[row.name] = row.role_id if str(row.role_id) not in valid_role_ids: valid_role_ids.extend([str(row.role_id), row.name]) try: if not all(rid.strip() in valid_role_ids for rid in choices): raise ValueError except ValueError: raise _errors.CredentialError("Found invalid role.") # Only return role IDs result = [] for rid in choices: try: result.append(int(rid)) except ValueError: # We got name result.append(roles[rid.strip()]) return result
def test_node_view(self): """Test basic properties/methods in the MySQLHandler. """ # Retrieve information on Fabric node. Note though # that there is no specific view to retrieve such # information. node_id_1 = _uuid.uuid4() node_startup_1 = _utils.get_time() _LOGGER.debug("Fabric Node started.", extra={ 'subject' : str(node_id_1), 'category' : MySQLHandler.NODE, 'type' : MySQLHandler.START, 'reported' : node_startup_1, } ) node_stop_1 = _utils.get_time() _LOGGER.debug("Fabric Node started.", extra={ 'subject' : str(node_id_1), 'category' : MySQLHandler.NODE, 'type' : MySQLHandler.STOP, 'reported' : node_stop_1, } ) node_id_2 = _uuid.uuid4() node_startup_2 = _utils.get_time() _LOGGER.debug("Fabric Node started.", extra={ 'subject' : str(node_id_2), 'category' : MySQLHandler.NODE, 'type' : MySQLHandler.START, 'reported' : node_startup_2, } ) node_view = ("SELECT subject as node_id, " "TIMEDIFF(UTC_TIMESTAMP(), reported) as node_uptime, " "reported as node_startup FROM log WHERE category = %s " "and type = %s ORDER BY node_id, node_startup" ) persister = _persistence.current_persister() res = persister.exec_stmt( node_view, { "params" : ( MySQLHandler.idx_category(MySQLHandler.NODE), MySQLHandler.idx_type(MySQLHandler.START) ) } ) self.assertEqual(len(res), 2)
def register(jobs, transaction): """Atomically register jobs. :param jobs: List of jobs to be registered. :param transaction: Whether there is transaction context or not. """ assert(isinstance(jobs, list)) persister = _persistence.current_persister() if not transaction: persister.begin() try: for job in jobs: if job.is_recoverable: job.checkpoint.register() except _errors.DatabaseError: try: if not transaction: persister.rollback() except _errors.DatabaseError as error: _LOGGER.error( "Error rolling back registered jobs.", exc_info=error ) raise else: try: # Currently, if the commit fails, we are not sure whether the # changes have succeeded or not. This is something that needs # to be improved in the near future. if not transaction: persister.commit() except _errors.DatabaseError as error: _LOGGER.error( "Error committing registered jobs.", exc_info=error ) raise
def register(jobs, transaction): """Atomically register jobs. :param jobs: List of jobs to be registered. :param transaction: Whether there is transaction context or not. """ assert (isinstance(jobs, list)) persister = _persistence.current_persister() if not transaction: persister.begin() try: for job in jobs: if job.is_recoverable: job.checkpoint.register() except _errors.DatabaseError: try: if not transaction: persister.rollback() except _errors.DatabaseError as error: _LOGGER.error("Error rolling back registered jobs.", exc_info=error) raise else: try: # Currently, if the commit fails, we are not sure whether the # changes have succeeded or not. This is something that needs # to be improved in the near future. if not transaction: persister.commit() except _errors.DatabaseError as error: _LOGGER.error("Error committing registered jobs.", exc_info=error) raise
def execute(self): """Display list of users """ persister = _persistence.current_persister() options = { "fetch": False, "columns": True, } role_perms = ( "SELECT u.username, u.protocol, r.name as role_name, " "r.description AS role_desc," "p.permission_id, p.description AS perm_desc, p.subsystem, " "p.component, p.function " "FROM users as u LEFT JOIN user_roles AS ur USING (user_id) " "LEFT JOIN roles AS r USING (role_id) " "LEFT JOIN role_permissions USING (role_id) " "LEFT JOIN permissions AS p USING (permission_id) " "ORDER BY u.username, u.protocol" ) cur = persister.exec_stmt(role_perms, options) roles = {} max_username_len = 0 max_protocol_len = 0 user_info = {} for row in cur: if len(row.username) > max_username_len: max_username_len = len(row.username) if len(row.protocol) > max_protocol_len: max_protocol_len = len(row.protocol) user_tuple = (row.username, row.protocol) if user_tuple not in user_info: user_info[user_tuple] = [] if row.role_name and row.role_name not in user_info[user_tuple]: user_info[user_tuple].append(row.role_name) # Minimum sizes max_username_len = max(9, max_username_len) max_protocol_len = max(12, max_protocol_len) role_fmt = ("{{username:{userlen}}} {{protocol:{protlen}}} " "{{roles}}".format(userlen=max_username_len, protlen=max_protocol_len)) header = role_fmt.format(username="******", protocol="Protocol", roles="Roles") print header print role_fmt.format(username='******' * max_username_len, protocol='-' * max_protocol_len, roles='-' * 20) for user_tuple, roles in user_info.iteritems(): username, protocol = user_tuple if roles: role_list = ', '.join(roles) else: role_list = '(no roles set)' print role_fmt.format(username=username, protocol=protocol, roles=role_list)
def test_persistence(self): """Test ErrorLog. """ # Update/Notify and fetch, they should match. interval = get_time_delta(1) now = get_time() input_whens = [ now, now ] input_reporters = [ "client:1000", "client:2000" ] st = ErrorLog(self.server, interval, now, input_whens, input_reporters) ErrorLog.add(self.server, now, "client:1000", "error") ErrorLog.add(self.server, now, "client:2000", "error") new_st = ErrorLog.fetch(self.server, interval, now) self.assertEqual(st.reporters, new_st.reporters) self.assertEqual(st.whens, new_st.whens) # Call remove, they should be empty and match. interval = get_time_delta(1) now = get_time() input_whens = [ ] input_reporters = [ ] ErrorLog.remove(self.server) st = ErrorLog(self.server, interval, now, input_whens, input_reporters) new_st = ErrorLog.fetch(self.server, interval, now) self.assertEqual(st.reporters, new_st.reporters) self.assertEqual(st.whens, new_st.whens) # Update/Notify and refresh, they should match. interval = get_time_delta(10) now = get_time() input_whens = [ now, now - get_time_delta(5) ] input_reporters = [ "client:1000", "client:2000" ] st = ErrorLog(self.server, interval, now, [], []) ErrorLog.add(self.server, now, "client:1000", "error") ErrorLog.add(self.server, now - get_time_delta(5), "client:2000", "error") ErrorLog.add(self.server, now - get_time_delta(11), "client:3000", "error") st.refresh() self.assertEqual(set(st.reporters), set(input_reporters)) self.assertEqual(set(st.whens), set(input_whens)) # Check whether a statement similar to the one used in the # event is fine. ErrorLog.remove(self.server) ErrorLog.add(self.server, now, "client:1000", "error") ErrorLog.add(self.server, now, "client:2000", "error") persister = _persistence.current_persister() out = persister.exec_stmt( "SELECT reported, UTC_TIMESTAMP() as now, " "TIMEDIFF(UTC_TIMESTAMP(), reported - MAKETIME(2,0,0)) as diff " "FROM error_log" ) _LOGGER.debug("Output test persistence %s.", out) self.assertEqual(len(out), 2) res = persister.exec_stmt( "DELETE FROM error_log WHERE " "TIMEDIFF(UTC_TIMESTAMP(), reported - MAKETIME(2,0,0)) > " "MAKETIME(1,0,0)" ) _LOGGER.debug("Output test persistence %s.", res) out = persister.exec_stmt( "SELECT reported, UTC_TIMESTAMP() as now, " "TIMEDIFF(UTC_TIMESTAMP(), reported - MAKETIME(2,0,0)) as diff " "FROM error_log" ) _LOGGER.debug("Output test persistence %s.", out) self.assertEqual(len(out), 0)
def _role_listing(selected=None, marker=None, persister=None): """Display list of roles :param selected: Sequence of selected roles. :param marker: Marker used for showing selected roles. The selected argument should be a list of role IDs. The marker will be shown in front of the role ID when it is in the selected sequence. """ if not persister: persister = _persistence.current_persister() if not selected: selected = [] if marker: marker = marker[0] else: marker = 'X' options = { "fetch": False, "columns": True, } roleperms = ( "SELECT r.role_id, r.name, r.description AS role_desc, " "p.permission_id, p.description AS perm_desc, p.subsystem, " "p.component, p.function " "FROM roles AS r LEFT JOIN role_permissions USING (role_id) " "LEFT JOIN permissions AS p USING (permission_id) ORDER BY r.role_id" ) cur = persister.exec_stmt(roleperms, options) roles = {} max_name_len = 0 max_rowid = 0 for row in cur: if len(row.name) > max_name_len: max_name_len = len(row.name) if row.role_id > max_rowid: max_rowid = row.role_id if row.role_id not in roles: roles[row.role_id] = [row.name, row.role_desc, [row.perm_desc]] else: roles[row.role_id][2].append(row.perm_desc) # Minimum sizes max_rowid_len = max(2, len(str(max_rowid))) max_name_len = max(9, max_name_len) sel = ' ' role_fmt = ("{{sel}} {{role_id:{idlen}}} {{name:{namelen}}}" " {{desc}}".format(idlen=max_rowid_len, namelen=max_name_len)) label_desc = "Description and Permissions" header = role_fmt.format(role_id="ID", name="Role Name", desc=label_desc, sel=sel) print header print role_fmt.format(role_id='-' * max_rowid_len, name='-' * max_name_len, desc='-' * len(label_desc), sel=sel) fmt_perm = '{0}+ {{perm}}'.format( (2 + max_rowid_len + 2 + max_name_len + 2) * ' ') for role_id in sorted(roles.keys()): if role_id in selected: sel = marker else: sel = ' ' name, role_desc, permissions = roles[role_id] print role_fmt.format(role_id=role_id, name=name, desc=role_desc, sel=sel) for perm in permissions: print fmt_perm.format(perm=perm)
def _role_listing(selected=None, marker=None, persister=None): """Display list of roles :param selected: Sequence of selected roles. :param marker: Marker used for showing selected roles. The selected argument should be a list of role IDs. The marker will be shown in front of the role ID when it is in the selected sequence. """ if not persister: persister = _persistence.current_persister() if not selected: selected = [] if marker: marker = marker[0] else: marker = 'X' options = { "fetch": False, "columns": True, } roleperms = ( "SELECT r.role_id, r.name, r.description AS role_desc, " "p.permission_id, p.description AS perm_desc, p.subsystem, " "p.component, p.function " "FROM roles AS r LEFT JOIN role_permissions USING (role_id) " "LEFT JOIN permissions AS p USING (permission_id) ORDER BY r.role_id") cur = persister.exec_stmt(roleperms, options) roles = {} max_name_len = 0 max_rowid = 0 for row in cur: if len(row.name) > max_name_len: max_name_len = len(row.name) if row.role_id > max_rowid: max_rowid = row.role_id if row.role_id not in roles: roles[row.role_id] = [row.name, row.role_desc, [row.perm_desc]] else: roles[row.role_id][2].append(row.perm_desc) # Minimum sizes max_rowid_len = max(2, len(str(max_rowid))) max_name_len = max(9, max_name_len) sel = ' ' role_fmt = ("{{sel}} {{role_id:{idlen}}} {{name:{namelen}}}" " {{desc}}".format(idlen=max_rowid_len, namelen=max_name_len)) label_desc = "Description and Permissions" header = role_fmt.format(role_id="ID", name="Role Name", desc=label_desc, sel=sel) print header print role_fmt.format(role_id='-' * max_rowid_len, name='-' * max_name_len, desc='-' * len(label_desc), sel=sel) fmt_perm = '{0}+ {{perm}}'.format( (2 + max_rowid_len + 2 + max_name_len + 2) * ' ') for role_id in sorted(roles.keys()): if role_id in selected: sel = marker else: sel = ' ' name, role_desc, permissions = roles[role_id] print role_fmt.format(role_id=role_id, name=name, desc=role_desc, sel=sel) for perm in permissions: print fmt_perm.format(perm=perm)
def execute(self): """Display list of users """ persister = _persistence.current_persister() options = { "fetch": False, "columns": True, } role_perms = ( "SELECT u.username, u.protocol, r.name as role_name, " "r.description AS role_desc," "p.permission_id, p.description AS perm_desc, p.subsystem, " "p.component, p.function " "FROM users as u LEFT JOIN user_roles AS ur USING (user_id) " "LEFT JOIN roles AS r USING (role_id) " "LEFT JOIN role_permissions USING (role_id) " "LEFT JOIN permissions AS p USING (permission_id) " "ORDER BY u.username, u.protocol") cur = persister.exec_stmt(role_perms, options) roles = {} max_username_len = 0 max_protocol_len = 0 user_info = {} for row in cur: if len(row.username) > max_username_len: max_username_len = len(row.username) if len(row.protocol) > max_protocol_len: max_protocol_len = len(row.protocol) user_tuple = (row.username, row.protocol) if user_tuple not in user_info: user_info[user_tuple] = [] if row.role_name and row.role_name not in user_info[user_tuple]: user_info[user_tuple].append(row.role_name) # Minimum sizes max_username_len = max(9, max_username_len) max_protocol_len = max(12, max_protocol_len) role_fmt = ("{{username:{userlen}}} {{protocol:{protlen}}} " "{{roles}}".format(userlen=max_username_len, protlen=max_protocol_len)) header = role_fmt.format(username="******", protocol="Protocol", roles="Roles") print header print role_fmt.format(username='******' * max_username_len, protocol='-' * max_protocol_len, roles='-' * 20) for user_tuple, roles in user_info.iteritems(): username, protocol = user_tuple if roles: role_list = ', '.join(roles) else: role_list = '(no roles set)' print role_fmt.format(username=username, protocol=protocol, roles=role_list)
def setUp(self): """Configure the existing environment """ self.persister = _persistence.current_persister() assert(self.persister is not None)
def test_persistence(self): """Test ErrorLog. """ # Update/Notify and fetch, they should match. interval = get_time_delta(1) now = get_time() input_whens = [now, now] input_reporters = ["client:1000", "client:2000"] st = ErrorLog(self.server, interval, now, input_whens, input_reporters) ErrorLog.add(self.server, now, "client:1000", "error") ErrorLog.add(self.server, now, "client:2000", "error") new_st = ErrorLog.fetch(self.server, interval, now) self.assertEqual(st.reporters, new_st.reporters) self.assertEqual(st.whens, new_st.whens) # Call remove, they should be empty and match. interval = get_time_delta(1) now = get_time() input_whens = [] input_reporters = [] ErrorLog.remove(self.server) st = ErrorLog(self.server, interval, now, input_whens, input_reporters) new_st = ErrorLog.fetch(self.server, interval, now) self.assertEqual(st.reporters, new_st.reporters) self.assertEqual(st.whens, new_st.whens) # Update/Notify and refresh, they should match. interval = get_time_delta(10) now = get_time() input_whens = [now, now - get_time_delta(5)] input_reporters = ["client:1000", "client:2000"] st = ErrorLog(self.server, interval, now, [], []) ErrorLog.add(self.server, now, "client:1000", "error") ErrorLog.add(self.server, now - get_time_delta(5), "client:2000", "error") ErrorLog.add(self.server, now - get_time_delta(11), "client:3000", "error") st.refresh() self.assertEqual(set(st.reporters), set(input_reporters)) self.assertEqual(set(st.whens), set(input_whens)) # Check whether a statement similar to the one used in the # event is fine. ErrorLog.remove(self.server) ErrorLog.add(self.server, now, "client:1000", "error") ErrorLog.add(self.server, now, "client:2000", "error") persister = _persistence.current_persister() out = persister.exec_stmt( "SELECT reported, UTC_TIMESTAMP() as now, " "TIMEDIFF(UTC_TIMESTAMP(), reported - MAKETIME(2,0,0)) as diff " "FROM error_log") _LOGGER.debug("Output test persistence %s.", out) self.assertEqual(len(out), 2) res = persister.exec_stmt( "DELETE FROM error_log WHERE " "TIMEDIFF(UTC_TIMESTAMP(), reported - MAKETIME(2,0,0)) > " "MAKETIME(1,0,0)") _LOGGER.debug("Output test persistence %s.", res) out = persister.exec_stmt( "SELECT reported, UTC_TIMESTAMP() as now, " "TIMEDIFF(UTC_TIMESTAMP(), reported - MAKETIME(2,0,0)) as diff " "FROM error_log") _LOGGER.debug("Output test persistence %s.", out) self.assertEqual(len(out), 0)