class ClockWithThreads(Clock): """ A testing reactor that supplies L{IReactorTime} and L{IReactorThreads}. """ def __init__(self): super(ClockWithThreads, self).__init__() self._pool = ThreadPool() def getThreadPool(self): """ Get the threadpool. """ return self._pool def suggestThreadPoolSize(self, size): """ Approximate the behavior of a "real" reactor. """ self._pool.adjustPoolsize(maxthreads=size) def callInThread(self, thunk, *a, **kw): """ No implementation. """ def callFromThread(self, thunk, *a, **kw): """
class ClockWithThreads(Clock): """ A testing reactor that supplies L{IReactorTime} and L{IReactorThreads}. """ def __init__(self): super(ClockWithThreads, self).__init__() self._pool = ThreadPool() def getThreadPool(self): """ Get the threadpool. """ return self._pool def suggestThreadPoolSize(self, size): """ Approximate the behavior of a 'real' reactor. """ self._pool.adjustPoolsize(maxthreads=size) def callInThread(self, thunk, *a, **kw): """ No implementation. """ def callFromThread(self, thunk, *a, **kw): """
def __init__( self, application, port=80, resources=None, services=None, loud=False): service.MultiService.__init__(self) # Create, start and add a thread pool service, which is made available # to our WSGIResource within HendrixResource threads = ThreadPool(name="Hendrix Service") # Testing threads 1-2-3 threads.adjustPoolsize(3, 5) reactor.addSystemEventTrigger('after', 'shutdown', threads.stop) ThreadPoolService(threads).setServiceParent(self) # create the base resource and add any additional static resources resource = HendrixResource(reactor, threads, application, loud=loud) if resources: resources = sorted(resources, key=lambda r: r.namespace) for res in resources: if hasattr(res, 'get_resources'): for sub_res in res.get_resources(): resource.putNamedChild(sub_res) else: resource.putNamedChild(res) factory = server.Site(resource) # add a tcp server that binds to port=port main_web_tcp = TCPServer(port, factory) main_web_tcp.setName('main_web_tcp') # to get this at runtime use # hedrix_service.getServiceNamed('main_web_tcp') main_web_tcp.setServiceParent(self) # add any additional services if services: for srv_name, srv in services: srv.setName(srv_name) srv.setServiceParent(self)
class DirectoryService(BaseDirectoryService): """ LDAP directory service. """ log = Logger() fieldName = ConstantsContainer((BaseFieldName, FieldName)) recordType = ConstantsContainer(( BaseRecordType.user, BaseRecordType.group, )) def __init__( self, url, baseDN, credentials=None, timeout=None, tlsCACertificateFile=None, tlsCACertificateDirectory=None, useTLS=False, fieldNameToAttributesMap=DEFAULT_FIELDNAME_ATTRIBUTE_MAP, recordTypeSchemas=DEFAULT_RECORDTYPE_SCHEMAS, extraFilters=None, ownThreadpool=True, threadPoolMax=10, authConnectionMax=5, queryConnectionMax=5, tries=3, warningThresholdSeconds=5, _debug=False, ): """ @param url: The URL of the LDAP server to connect to. @type url: L{unicode} @param baseDN: The base DN for queries. @type baseDN: L{unicode} @param credentials: The credentials to use to authenticate with the LDAP server. @type credentials: L{IUsernamePassword} @param timeout: A timeout, in seconds, for LDAP queries. @type timeout: number @param tlsCACertificateFile: ... @type tlsCACertificateFile: L{FilePath} @param tlsCACertificateDirectory: ... @type tlsCACertificateDirectory: L{FilePath} @param useTLS: Enable the use of TLS. @type useTLS: L{bool} @param fieldNameToAttributesMap: A mapping of field names to LDAP attribute names. @type fieldNameToAttributesMap: mapping with L{NamedConstant} keys and sequence of L{unicode} values @param recordTypeSchemas: Schema information for record types. @type recordTypeSchemas: mapping from L{NamedConstant} to L{RecordTypeSchema} @param extraFilters: A dict (keyed off recordType) of extra filter fragments to AND in to any generated queries. @type extraFilters: L{dicts} of L{unicode} """ self.url = url self._baseDN = baseDN self._credentials = credentials self._timeout = timeout self._extraFilters = extraFilters self._tries = tries self._warningThresholdSeconds = warningThresholdSeconds if tlsCACertificateFile is None: self._tlsCACertificateFile = None else: self._tlsCACertificateFile = tlsCACertificateFile.path if tlsCACertificateDirectory is None: self._tlsCACertificateDirectory = None else: self._tlsCACertificateDirectory = tlsCACertificateDirectory.path self._useTLS = useTLS if _debug: self._debug = 255 else: self._debug = None if self.fieldName.recordType in fieldNameToAttributesMap: raise TypeError("Record type field may not be mapped") if BaseFieldName.uid not in fieldNameToAttributesMap: raise DirectoryConfigurationError("Mapping for uid required") self._fieldNameToAttributesMap = fieldNameToAttributesMap self._attributeToFieldNameMap = {} for name, attributes in fieldNameToAttributesMap.iteritems(): for attribute in attributes: if ":" in attribute: attribute, ignored = attribute.split(":", 1) self._attributeToFieldNameMap.setdefault(attribute, []).append(name) self._recordTypeSchemas = recordTypeSchemas attributesToFetch = set() for attributes in fieldNameToAttributesMap.values(): for attribute in attributes: if ":" in attribute: attribute, ignored = attribute.split(":", 1) attributesToFetch.add(attribute.encode("utf-8")) self._attributesToFetch = list(attributesToFetch) # Threaded connection pool. # The connection size limit here is the size for connections doing # queries. # There will also be one-off connections for authentications which also # run in their own threads. # Thus the threadpool max ought to be larger than the connection max to # allow for both pooled query connections and one-off auth-only # connections. self.ownThreadpool = ownThreadpool if self.ownThreadpool: self.threadpool = ThreadPool( minthreads=1, maxthreads=threadPoolMax, name="LDAPDirectoryService", ) else: # Use the default threadpool but adjust its size to fit our needs self.threadpool = reactor.getThreadPool() self.threadpool.adjustPoolsize( max(threadPoolMax, self.threadpool.max)) # Separate pools for LDAP queries and LDAP binds. self.connectionPools = { "query": ConnectionPool("query", self, credentials, queryConnectionMax), "auth": ConnectionPool("auth", self, None, authConnectionMax), } self.poolStats = collections.defaultdict(int) reactor.callWhenRunning(self.start) reactor.addSystemEventTrigger("during", "shutdown", self.stop) def getPreferredRecordTypesOrder(self): # Not doing this in init( ) because we get our recordTypes assigned later if not hasattr(self, "_preferredRecordTypesOrder"): self._preferredRecordTypesOrder = [] for recordTypeName in [ "user", "location", "resource", "group", "address" ]: try: recordType = self.recordType.lookupByName(recordTypeName) self._preferredRecordTypesOrder.append(recordType) except ValueError: pass return self._preferredRecordTypesOrder def start(self): """ Start up this service. Initialize the threadpool (if we own it). """ if self.ownThreadpool: self.threadpool.start() def stop(self): """ Stop the service. Stop the threadpool if we own it and do other clean-up. """ if self.ownThreadpool: self.threadpool.stop() # FIXME: we should probably also close the pool of active connections # too. @property def realmName(self): return u"{self.url}".format(self=self) class Connection(object): """ ContextManager object for getting a connection from the pool. On exit the connection will be put back in the pool if no exception was raised. Otherwise, the connection will be removed from the active connection list, which will allow a new "clean" connection to be created later if needed. """ def __init__(self, ds, poolName): self.pool = ds.connectionPools[poolName] def __enter__(self): self.connection = self.pool.getConnection() return self.connection def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: self.pool.returnConnection(self.connection) return True else: self.pool.failedConnection(self.connection) return False def _authenticateUsernamePassword(self, dn, password): """ Open a secondary connection to the LDAP server and try binding to it with the given credentials @returns: True if the password is correct, False otherwise @rtype: deferred C{bool} @raises: L{LDAPConnectionError} if unable to connect. """ d = deferToThreadPool(reactor, self.threadpool, self._authenticateUsernamePassword_inThread, dn, password) qsize = self.threadpool._queue.qsize() if qsize > 0: self.log.error("LDAP thread pool overflowing: {qsize}", qsize=qsize) self.poolStats["connection-thread-blocked"] += 1 return d def _authenticateUsernamePassword_inThread(self, dn, password, testStats=None): """ Open a secondary connection to the LDAP server and try binding to it with the given credentials. This method is always called in a thread. @returns: True if the password is correct, False otherwise @rtype: C{bool} @raises: L{LDAPConnectionError} if unable to connect. """ self.log.debug("Authenticating {dn}", dn=dn) # Retry if we get ldap.SERVER_DOWN for retryNumber in xrange(self._tries): # For unit tests, a bit of instrumentation so we can examine # retryNumber: if testStats is not None: testStats["retryNumber"] = retryNumber try: with DirectoryService.Connection(self, "auth") as connection: try: # During testing, allow an exception to be raised. # Note: I tried to use patch( ) to accomplish this # but that seemed to create a race condition in the # restoration of the patched value and that would cause # unit tests to occasionally fail. if testStats is not None: if "raise" in testStats: raise testStats["raise"] connection.simple_bind_s(dn, password) self.log.debug("Authenticated {dn}", dn=dn) return True except ( ldap.INAPPROPRIATE_AUTH, ldap.INVALID_CREDENTIALS, ldap.INVALID_DN_SYNTAX, ): self.log.debug("Unable to authenticate {dn}", dn=dn) return False except ldap.CONSTRAINT_VIOLATION: self.log.info("Account locked {dn}", dn=dn) return False except ldap.SERVER_DOWN as e: # Catch this below for retry raise e except Exception as e: self.log.error( "Unexpected error {error} trying to authenticate {dn}", error=str(e), dn=dn) return False else: # Do an unauthenticated bind on this connection at the end in # case the server limits the number of concurrent auths by a given user. connection.simple_bind_s("", "") except ldap.SERVER_DOWN as e: self.log.error("LDAP server unavailable") if retryNumber + 1 == self._tries: # We've hit SERVER_DOWN self._tries times, giving up. raise LDAPQueryError("LDAP server down", e) else: self.log.error("LDAP connection failure; retrying...") def _recordsFromQueryString(self, queryString, recordTypes=None, limitResults=None, timeoutSeconds=None): d = deferToThreadPool(reactor, self.threadpool, self._recordsFromQueryString_inThread, queryString, recordTypes, limitResults=limitResults, timeoutSeconds=timeoutSeconds) qsize = self.threadpool._queue.qsize() if qsize > 0: self.log.error("LDAP thread pool overflowing: {qsize}", qsize=qsize) self.poolStats["connection-thread-blocked"] += 1 return d def _addExtraFilter(self, recordType, queryString): if self._extraFilters and self._extraFilters.get(recordType, ""): queryString = u"(&{extra}{query})".format( extra=self._extraFilters[recordType], query=queryString) return queryString def _recordsFromQueryString_inThread(self, queryString, recordTypes=None, limitResults=None, timeoutSeconds=None, testStats=None): # This method is always called in a thread. if recordTypes is None: # recordTypes = list(self.recordTypes()) # Quick hack to optimize the order in which we query by record type: recordTypes = self.getPreferredRecordTypesOrder() # Retry if we get ldap.SERVER_DOWN for retryNumber in xrange(self._tries): # For unit tests, a bit of instrumentation so we can examine # retryNumber: if testStats is not None: testStats["retryNumber"] = retryNumber records = [] try: with DirectoryService.Connection(self, "query") as connection: for recordType in recordTypes: if limitResults is not None: if limitResults < 1: break try: rdn = self._recordTypeSchemas[ recordType].relativeDN except KeyError: # Skip this unknown record type continue rdn = (ldap.dn.str2dn(rdn.lower()) + ldap.dn.str2dn(self._baseDN.lower())) filteredQuery = self._addExtraFilter( recordType, queryString) self.log.debug( "Performing LDAP query: " "{rdn} {query} {recordType}{limit}{timeout}", rdn=rdn, query=filteredQuery.encode("utf-8"), recordType=recordType, limit=(" limit={}".format(limitResults) if limitResults else ""), timeout=(" timeout={}".format(timeoutSeconds) if timeoutSeconds else ""), ) try: startTime = time.time() s = ldap. async .List(connection) s.startSearch( ldap.dn.dn2str(rdn), ldap.SCOPE_SUBTREE, filteredQuery.encode("utf-8"), attrList=self._attributesToFetch, timeout=(timeoutSeconds if timeoutSeconds else -1), sizelimit=(limitResults if limitResults else 0), ) s.processResults() except ldap.SIZELIMIT_EXCEEDED as e: self.log.debug( "LDAP result limit exceeded: {limit}", limit=limitResults, ) except ldap.TIMELIMIT_EXCEEDED as e: self.log.warn( "LDAP timeout exceeded: {timeout} seconds", timeout=timeoutSeconds, ) except ldap.FILTER_ERROR as e: self.log.error( "Unable to perform query {query!r}: {err}", query=queryString, err=e) raise LDAPQueryError("Unable to perform query", e) except ldap.NO_SUCH_OBJECT as e: # self.log.warn( # "RDN {rdn} does not exist, skipping", rdn=rdn # ) continue except ldap.INVALID_SYNTAX as e: self.log.error( "LDAP invalid syntax {query!r}: {err}", query=queryString, err=e) continue except ldap.SERVER_DOWN as e: # Catch this below for retry raise e except Exception as e: self.log.error("LDAP error {query!r}: {err}", query=queryString, err=e) raise LDAPQueryError("Unable to perform query", e) reply = [ resultItem for _ignore_resultType, resultItem in s.allResults ] totalTime = time.time() - startTime if totalTime > self._warningThresholdSeconds: if filteredQuery and len(filteredQuery) > 500: filteredQuery = "%s..." % ( filteredQuery[:500], ) self.log.error( "LDAP query exceeded threshold: {totalTime:.2f} seconds for {rdn} {query} (#results={resultCount})", totalTime=totalTime, rdn=rdn, query=filteredQuery, resultCount=len(reply)) newRecords = self._recordsFromReply( reply, recordType=recordType) self.log.debug( "Records from LDAP query " "({rdn} {query} {recordType}): {count}", rdn=rdn, query=queryString, recordType=recordType, count=len(newRecords)) if limitResults is not None: limitResults = limitResults - len(newRecords) records.extend(newRecords) except ldap.SERVER_DOWN as e: self.log.error("LDAP server unavailable") if retryNumber + 1 == self._tries: # We've hit SERVER_DOWN self._tries times, giving up. raise LDAPQueryError("LDAP server down", e) else: self.log.error("LDAP connection failure; retrying...") else: # Only retry if we got ldap.SERVER_DOWN, otherwise break out of # loop. break self.log.debug("LDAP result count ({query}): {count}", query=queryString, count=len(records)) return records def _recordWithDN(self, dn): d = deferToThreadPool(reactor, self.threadpool, self._recordWithDN_inThread, dn) qsize = self.threadpool._queue.qsize() if qsize > 0: self.log.error("LDAP thread pool overflowing: {qsize}", qsize=qsize) self.poolStats["connection-thread-blocked"] += 1 return d def _recordWithDN_inThread(self, dn, testStats=None): """ @param dn: The DN of the record to search for @type dn: C{str} """ # This method is always called in a thread. records = [] # Retry if we get ldap.SERVER_DOWN for retryNumber in xrange(self._tries): # For unit tests, a bit of instrumentation: if testStats is not None: testStats["retryNumber"] = retryNumber try: with DirectoryService.Connection(self, "query") as connection: self.log.debug("Performing LDAP DN query: {dn}", dn=dn) try: reply = connection.search_s( dn, ldap.SCOPE_SUBTREE, "(objectClass=*)", attrlist=self._attributesToFetch) records = self._recordsFromReply(reply) except ldap.NO_SUCH_OBJECT: records = [] except ldap.INVALID_DN_SYNTAX: self.log.warn("Invalid LDAP DN syntax: '{dn}'", dn=dn) records = [] except ldap.SERVER_DOWN as e: self.log.error("LDAP server unavailable") if retryNumber + 1 == self._tries: # We've hit SERVER_DOWN self._tries times, giving up raise LDAPQueryError("LDAP server down", e) else: self.log.error("LDAP connection failure; retrying...") else: # Only retry if we got ldap.SERVER_DOWN, otherwise break out of # loop break if len(records): return records[0] else: return None def _recordsFromReply(self, reply, recordType=None): records = [] for dn, recordData in reply: # Determine the record type if recordType is None: recordType = recordTypeForDN(self._baseDN, self._recordTypeSchemas, dn) if recordType is None: recordType = recordTypeForRecordData(self._recordTypeSchemas, recordData) if recordType is None: self.log.debug( "Ignoring LDAP record data; unable to determine record " "type: {recordData!r}", recordData=recordData, ) continue # Populate a fields dictionary fields = {} for fieldName, attributeRules in ( self._fieldNameToAttributesMap.iteritems()): valueType = self.fieldName.valueType(fieldName) for attributeRule in attributeRules: attributeName = attributeRule.split(":")[0] if attributeName in recordData: values = recordData[attributeName] if valueType in (unicode, UUID): if not isinstance(values, list): values = [values] if valueType is unicode: newValues = [] for v in values: if isinstance(v, unicode): # because the ldap unit test produces # unicode values (?) newValues.append(v) else: try: newValues.append( unicode(v, "utf-8")) except UnicodeDecodeError: # Log and re-raise so the net behavior is as before during debugging self.log.error( "Received non-UTF-8 bytes from LDAP for {dn} in {name}", dn=dn, name=fieldName) raise else: try: newValues = [valueType(v) for v in values] except Exception, e: self.log.warn( "Can't parse value {name} {values} " "({error})", name=fieldName, values=values, error=str(e)) continue if self.fieldName.isMultiValue(fieldName): if fieldName in fields: fields[fieldName].extend(newValues) else: fields[fieldName] = newValues else: # First one in the list wins if fieldName not in fields: fields[fieldName] = newValues[0] elif valueType is bool: if not isinstance(values, list): values = [values] if ":" in attributeRule: ignored, trueValue = attributeRule.split(":") else: trueValue = "true" for value in values: if value == trueValue: fields[fieldName] = True break else: fields[fieldName] = False elif issubclass(valueType, Names): if not isinstance(values, list): values = [values] _ignore_attribute, attributeValue, fieldValue = ( attributeRule.split(":")) for value in values: if value == attributeValue: # convert to a constant try: fieldValue = ( valueType.lookupByName(fieldValue)) fields[fieldName] = fieldValue except ValueError: pass break else: raise LDAPConfigurationError( "Unknown value type {0} for field {1}".format( valueType, fieldName)) # Skip any results missing the uid, which is a required field if self.fieldName.uid not in fields: continue # Set record type and dn fields fields[self.fieldName.recordType] = recordType fields[self.fieldName.dn] = dn.decode("utf-8") # Make a record object from fields. record = DirectoryRecord(self, fields) records.append(record) # self.log.debug("LDAP results: {records}", records=records) return records
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # from protocol import format_string from twisted.python.threadpool import ThreadPool import logging import time logger = logging.getLogger(name='plugin') pool = ThreadPool() pool.adjustPoolsize(1) pool.start() class Plugin: """ Base class of a plugin. Use this when writing new plugins. Provides several methods to communicate with the clients. Provide at least these methods: - name(): String ID of the plugin. It must match the name in the client counterpart. - message_from_client(message): Called when the client sends some data, usually as a response to some request. """ def __init__(self, plugins):
class DirectoryService(BaseDirectoryService): """ LDAP directory service. """ log = Logger() fieldName = ConstantsContainer((BaseFieldName, FieldName)) recordType = ConstantsContainer(( BaseRecordType.user, BaseRecordType.group, )) def __init__( self, url, baseDN, credentials=None, timeout=None, tlsCACertificateFile=None, tlsCACertificateDirectory=None, useTLS=False, fieldNameToAttributesMap=DEFAULT_FIELDNAME_ATTRIBUTE_MAP, recordTypeSchemas=DEFAULT_RECORDTYPE_SCHEMAS, extraFilters=None, ownThreadpool=True, threadPoolMax=10, connectionMax=10, tries=3, _debug=False, ): """ @param url: The URL of the LDAP server to connect to. @type url: L{unicode} @param baseDN: The base DN for queries. @type baseDN: L{unicode} @param credentials: The credentials to use to authenticate with the LDAP server. @type credentials: L{IUsernamePassword} @param timeout: A timeout, in seconds, for LDAP queries. @type timeout: number @param tlsCACertificateFile: ... @type tlsCACertificateFile: L{FilePath} @param tlsCACertificateDirectory: ... @type tlsCACertificateDirectory: L{FilePath} @param useTLS: Enable the use of TLS. @type useTLS: L{bool} @param fieldNameToAttributesMap: A mapping of field names to LDAP attribute names. @type fieldNameToAttributesMap: mapping with L{NamedConstant} keys and sequence of L{unicode} values @param recordTypeSchemas: Schema information for record types. @type recordTypeSchemas: mapping from L{NamedConstant} to L{RecordTypeSchema} @param extraFilters: A dict (keyed off recordType) of extra filter fragments to AND in to any generated queries. @type extraFilters: L{dicts} of L{unicode} """ self.url = url self._baseDN = baseDN self._credentials = credentials self._timeout = timeout self._extraFilters = extraFilters self._tries = tries if tlsCACertificateFile is None: self._tlsCACertificateFile = None else: self._tlsCACertificateFile = tlsCACertificateFile.path if tlsCACertificateDirectory is None: self._tlsCACertificateDirectory = None else: self._tlsCACertificateDirectory = tlsCACertificateDirectory.path self._useTLS = useTLS if _debug: self._debug = 255 else: self._debug = None if self.fieldName.recordType in fieldNameToAttributesMap: raise TypeError("Record type field may not be mapped") if BaseFieldName.uid not in fieldNameToAttributesMap: raise DirectoryConfigurationError("Mapping for uid required") self._fieldNameToAttributesMap = fieldNameToAttributesMap self._attributeToFieldNameMap = {} for name, attributes in fieldNameToAttributesMap.iteritems(): for attribute in attributes: if ":" in attribute: attribute, ignored = attribute.split(":", 1) self._attributeToFieldNameMap.setdefault(attribute, []).append(name) self._recordTypeSchemas = recordTypeSchemas attributesToFetch = set() for attributes in fieldNameToAttributesMap.values(): for attribute in attributes: if ":" in attribute: attribute, ignored = attribute.split(":", 1) attributesToFetch.add(attribute.encode("utf-8")) self._attributesToFetch = list(attributesToFetch) # Threaded connection pool. The connection size limit here is the size for connections doing queries. # There will also be one-off connections for authentications which also run in their own threads. Thus # the threadpool max ought to be larger than the connection max to allow for both pooled query connections # and one-off auth-only connections. self.ownThreadpool = ownThreadpool if self.ownThreadpool: self.threadpool = ThreadPool(minthreads=1, maxthreads=threadPoolMax, name="LDAPDirectoryService") else: # Use the default threadpool but adjust its size to fit our needs self.threadpool = reactor.getThreadPool() self.threadpool.adjustPoolsize( max(threadPoolMax, self.threadpool.max)) self.connectionMax = connectionMax self.connectionCreateLock = RLock() self.connections = [] self.connectionQueue = Queue() self.poolStats = collections.defaultdict(int) self.activeCount = 0 reactor.callWhenRunning(self.start) reactor.addSystemEventTrigger('during', 'shutdown', self.stop) def start(self): """ Start up this service. Initialize the threadpool (if we own it). """ if self.ownThreadpool: self.threadpool.start() def stop(self): """ Stop the service. Stop the threadpool if we own it and do other clean-up. """ if self.ownThreadpool: self.threadpool.stop() # FIXME: we should probably also close the pool of active connections too @property def realmName(self): return u"{self.url}".format(self=self) class Connection(object): """ ContextManager object for getting a connection from the pool. On exit the connection will be put back in the pool if no exception was raised. Otherwise, the connection will be removed from the active connection list, which will allow a new "clean" connection to be created later if needed. """ def __init__(self, ds): self.ds = ds def __enter__(self): self.connection = self.ds._getConnection() return self.connection def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: self.ds._returnConnection(self.connection) return True else: self.ds._failedConnection(self.connection) return False def _getConnection(self): """ Get a connection from the connection pool. This will retrieve a connection from the connection pool L{Queue} object. If the L{Queue} is empty, it will check to see whether a new connection can be created (based on the connection limit), and if so create that and use it. If no new connections can be created, it will block on the L{Queue} until an existing, in-use, connection is put back. """ try: connection = self.connectionQueue.get(block=False) except Empty: # Note we use a lock here to prevent a race condition in which multiple requests for a new connection # could succeed even though the connection counts starts out one less than the maximum. This can happen # because self._connect() can take a while. self.connectionCreateLock.acquire() if len(self.connections) < self.connectionMax: connection = self._connect() self.connections.append(connection) self.connectionCreateLock.release() else: self.connectionCreateLock.release() self.poolStats["connection-blocked"] += 1 connection = self.connectionQueue.get() self.poolStats["connection-{}".format( self.connections.index(connection))] += 1 self.activeCount += 1 self.poolStats["connection-max"] = max( self.poolStats["connection-max"], self.activeCount) return connection def _returnConnection(self, connection): """ A connection is no longer needed - return it to the pool. """ self.activeCount -= 1 self.connectionQueue.put(connection) def _failedConnection(self, connection): """ A connection has failed - remove it from the list of active connections. A new one will be created if needed. """ self.activeCount -= 1 self.poolStats["connection-errors"] += 1 self.connections.remove(connection) def _connect(self): """ Connect to the directory server. This will always be called in a thread to prevent blocking. @returns: The connection object. @rtype: L{ldap.ldapobject.LDAPObject} @raises: L{LDAPConnectionError} if unable to connect. """ # FIXME: ldap connection objects are not thread safe, so let's set up # a connection pool self.log.debug("Connecting to LDAP at {log_source.url}") connection = self._newConnection() if self._credentials is not None: if IUsernamePassword.providedBy(self._credentials): try: connection.simple_bind_s( self._credentials.username, self._credentials.password, ) self.log.debug("Bound to LDAP as {credentials.username}", credentials=self._credentials) except (ldap.INVALID_CREDENTIALS, ldap.INVALID_DN_SYNTAX) as e: self.log.error( "Unable to bind to LDAP as {credentials.username}", credentials=self._credentials) raise LDAPBindAuthError(self._credentials.username, e) else: raise LDAPConnectionError( "Unknown credentials type: {0}".format(self._credentials)) return connection def _newConnection(self): """ Create a new LDAP connection and initialize and start TLS if required. This will always be called in a thread to prevent blocking. @returns: The connection object. @rtype: L{ldap.ldapobject.LDAPObject} @raises: L{LDAPConnectionError} if unable to connect. """ connection = ldap.initialize(self.url) # FIXME: Use trace_file option to wire up debug logging when # Twisted adopts the new logging stuff. for option, value in ( (ldap.OPT_TIMEOUT, self._timeout), (ldap.OPT_X_TLS_CACERTFILE, self._tlsCACertificateFile), (ldap.OPT_X_TLS_CACERTDIR, self._tlsCACertificateDirectory), (ldap.OPT_DEBUG_LEVEL, self._debug), ): if value is not None: connection.set_option(option, value) if self._useTLS: self.log.debug("Starting TLS for {log_source.url}") connection.start_tls_s() return connection def _authenticateUsernamePassword(self, dn, password): """ Open a secondary connection to the LDAP server and try binding to it with the given credentials @returns: True if the password is correct, False otherwise @rtype: deferred C{bool} @raises: L{LDAPConnectionError} if unable to connect. """ return deferToThreadPool(reactor, self.threadpool, self._authenticateUsernamePassword_inThread, dn, password) def _authenticateUsernamePassword_inThread(self, dn, password): """ Open a secondary connection to the LDAP server and try binding to it with the given credentials. This method is always called in a thread. @returns: True if the password is correct, False otherwise @rtype: C{bool} @raises: L{LDAPConnectionError} if unable to connect. """ self.log.debug("Authenticating {dn}", dn=dn) connection = self._newConnection() try: connection.simple_bind_s(dn, password) self.log.debug("Authenticated {dn}", dn=dn) return True except (ldap.INVALID_CREDENTIALS, ldap.INVALID_DN_SYNTAX): self.log.debug("Unable to authenticate {dn}", dn=dn) return False finally: # TODO: should we explicitly "close" the connection in a finally # clause rather than just let it go out of scope and be garbage collected # at some indeterminate point in the future? Up side is that we won't hang # on to the connection or other resources for longer than needed. Down side # is we will take a little bit of extra time in this call to close it down. # If we do decide to "close" then we probably have to use one of the "unbind" # methods on the L{LDAPObject}. connection = None def _recordsFromQueryString(self, queryString, recordTypes=None, limitResults=None, timeoutSeconds=None): return deferToThreadPool(reactor, self.threadpool, self._recordsFromQueryString_inThread, queryString, recordTypes, limitResults=limitResults, timeoutSeconds=timeoutSeconds) def _addExtraFilter(self, recordType, queryString): if self._extraFilters and self._extraFilters.get(recordType, ""): queryString = "(&{extra}{query})".format( extra=self._extraFilters[recordType], query=queryString) return queryString def _recordsFromQueryString_inThread(self, queryString, recordTypes=None, limitResults=None, timeoutSeconds=None): """ This method is always called in a thread. """ if recordTypes is None: recordTypes = list(self.recordTypes()) # Retry if we get ldap.SERVER_DOWN for self._retryNumber in xrange(self._tries): records = [] try: with DirectoryService.Connection(self) as connection: for recordType in recordTypes: if limitResults is not None: if limitResults < 1: break try: rdn = self._recordTypeSchemas[ recordType].relativeDN except KeyError: # Skip this unknown record type continue rdn = (ldap.dn.str2dn(rdn.lower()) + ldap.dn.str2dn(self._baseDN.lower())) filteredQuery = self._addExtraFilter( recordType, queryString) self.log.debug( "Performing LDAP query: {rdn} {query} {recordType}{limit}{timeout}", rdn=rdn, query=filteredQuery, recordType=recordType, limit=" limit={}".format(limitResults) if limitResults else "", timeout=" timeout={}".format(timeoutSeconds) if timeoutSeconds else "", ) try: s = ldap. async .List(connection) s.startSearch( ldap.dn.dn2str(rdn), ldap.SCOPE_SUBTREE, filteredQuery, attrList=self._attributesToFetch, timeout=timeoutSeconds if timeoutSeconds else -1, sizelimit=limitResults if limitResults else 0) s.processResults() except ldap.SIZELIMIT_EXCEEDED as e: self.log.debug( "LDAP result limit exceeded: {}".format( limitResults, )) except ldap.TIMELIMIT_EXCEEDED as e: self.log.warn( "LDAP timeout exceeded: {} seconds".format( timeoutSeconds, )) except ldap.FILTER_ERROR as e: self.log.error( "Unable to perform query {query!r}: {err}", query=queryString, err=e) raise LDAPQueryError("Unable to perform query", e) except ldap.NO_SUCH_OBJECT as e: # self.log.warn("RDN {rdn} does not exist, skipping", rdn=rdn) continue except ldap.INVALID_SYNTAX as e: self.log.error( "LDAP invalid syntax {query!r}: {err}", query=queryString, err=e) continue except ldap.SERVER_DOWN as e: # Catch this below for retry raise e except Exception as e: self.log.error("LDAP error {query!r}: {err}", query=queryString, err=e) raise LDAPQueryError("Unable to perform query", e) reply = [ resultItem for _ignore_resultType, resultItem in s.allResults ] newRecords = self._recordsFromReply( reply, recordType=recordType) self.log.debug( "Records from LDAP query ({rdn} {query} {recordType}): {count}", rdn=rdn, query=queryString, recordType=recordType, count=len(newRecords)) if limitResults is not None: limitResults = limitResults - len(newRecords) records.extend(newRecords) except ldap.SERVER_DOWN as e: self.log.error("LDAP server unavailable") if self._retryNumber + 1 == self._tries: # We've hit SERVER_DOWN self._tries times, giving up raise LDAPQueryError("LDAP server down", e) else: self.log.error("LDAP connection failure; retrying...") else: # Only retry if we got ldap.SERVER_DOWN, otherwise break out of # loop break self.log.debug("LDAP result count ({query}): {count}", query=queryString, count=len(records)) return records def _recordWithDN(self, dn): return deferToThreadPool(reactor, self.threadpool, self._recordWithDN_inThread, dn) def _recordWithDN_inThread(self, dn): """ This method is always called in a thread. @param dn: The DN of the record to search for @type dn: C{str} """ records = [] # Retry if we get ldap.SERVER_DOWN for self._retryNumber in xrange(self._tries): try: with DirectoryService.Connection(self) as connection: self.log.debug("Performing LDAP DN query: {dn}", dn=dn) try: reply = connection.search_s( dn, ldap.SCOPE_SUBTREE, "(objectClass=*)", attrlist=self._attributesToFetch) records = self._recordsFromReply(reply) except ldap.NO_SUCH_OBJECT: records = [] except ldap.INVALID_DN_SYNTAX: self.log.warn("Invalid LDAP DN syntax: '{dn}'", dn=dn) records = [] except ldap.SERVER_DOWN as e: self.log.error("LDAP server unavailable") if self._retryNumber + 1 == self._tries: # We've hit SERVER_DOWN self._tries times, giving up raise LDAPQueryError("LDAP server down", e) else: self.log.error("LDAP connection failure; retrying...") else: # Only retry if we got ldap.SERVER_DOWN, otherwise break out of # loop break if len(records): return records[0] else: return None def _recordsFromReply(self, reply, recordType=None): records = [] for dn, recordData in reply: # Determine the record type if recordType is None: recordType = recordTypeForDN(self._baseDN, self._recordTypeSchemas, dn) if recordType is None: recordType = recordTypeForRecordData(self._recordTypeSchemas, recordData) if recordType is None: self.log.debug( "Ignoring LDAP record data; unable to determine record " "type: {recordData!r}", recordData=recordData, ) continue # Populate a fields dictionary fields = {} for fieldName, attributeRules in self._fieldNameToAttributesMap.iteritems( ): valueType = self.fieldName.valueType(fieldName) for attributeRule in attributeRules: attributeName = attributeRule.split(":")[0] if attributeName in recordData: values = recordData[attributeName] if valueType in (unicode, UUID): if not isinstance(values, list): values = [values] if valueType is unicode: newValues = [] for v in values: if isinstance(v, unicode): # because the ldap unit test produces # unicode values (?) newValues.append(v) else: newValues.append(unicode(v, "utf-8")) else: try: newValues = [valueType(v) for v in values] except Exception, e: self.log.warn( "Can't parse value {name} {values} ({error})", name=fieldName, values=values, error=str(e)) continue if self.fieldName.isMultiValue(fieldName): if fieldName in fields: fields[fieldName].extend(newValues) else: fields[fieldName] = newValues else: # First one in the list wins if fieldName not in fields: fields[fieldName] = newValues[0] elif valueType is bool: if not isinstance(values, list): values = [values] if ":" in attributeRule: ignored, trueValue = attributeRule.split(":") else: trueValue = "true" for value in values: if value == trueValue: fields[fieldName] = True break else: fields[fieldName] = False elif issubclass(valueType, Names): if not isinstance(values, list): values = [values] _ignore_attribute, attributeValue, fieldValue = attributeRule.split( ":") for value in values: if value == attributeValue: # convert to a constant try: fieldValue = valueType.lookupByName( fieldValue) fields[fieldName] = fieldValue except ValueError: pass break else: raise LDAPConfigurationError( "Unknown value type {0} for field {1}".format( valueType, fieldName)) # Skip any results missing the uid, which is a required field if self.fieldName.uid not in fields: continue # Set record type and dn fields fields[self.fieldName.recordType] = recordType fields[self.fieldName.dn] = dn.decode("utf-8") # Make a record object from fields. record = DirectoryRecord(self, fields) records.append(record) # self.log.debug("LDAP results: {records}", records=records) return records
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # from protocol import format_string from twisted.python.threadpool import ThreadPool import logging import time logger = logging.getLogger(name='plugin') pool = ThreadPool() pool.adjustPoolsize(1) pool.start() class Plugin: """ Base class of a plugin. Use this when writing new plugins. Provides several methods to communicate with the clients. Provide at least these methods: - name(): String ID of the plugin. It must match the name in the client counterpart. - message_from_client(message): Called when the client sends some data, usually as a response to some request. """ def __init__(self, plugins): """
class Stats(object): """ stats.Stats rrdtool templated poller, updater, and web frontend """ # Our default configuration directives, if they don't # exist in the configuration file, they'll reflect what's here default_config = ImmutableDict(**dict( graphs = [], httpd_port = 8080, interface = '0.0.0.0', graph_draw_frequency = dict( hour = 60, day = 300, week = 300*3, default = 60 ), wsgi_min_threads = 1, wsgi_max_threads = 5 )) def __init__(self, instance_name): self.instance_name = instance_name self.config = Config(self.default_config) self.flask_app = WebApp(self) self.template_runner = TemplateRunner(self) self.active_graphs = dict() self.last_draw_timestamp = dict() self.wsgi_threadpool = ThreadPool(minthreads = self.config['wsgi_min_threads'], maxthreads=self.config['wsgi_max_threads'], name = 'wsgi_threadpool') def validate_config(self): """ Validates the loaded configuration. """ c = self.config # Make sure that we have a database_path, and an image_path... assert 'database_path' in c assert 'image_path' in c # We should probably check if these paths exist and make them as well... # Set the default values. graph_draw_frequency = c['graph_draw_frequency'] for period, interval in self.default_config['graph_draw_frequency'].iteritems(): graph_draw_frequency.setdefault(period, interval) # A quick check to make sure that our port is an integer. c['httpd_port'] = int(c['httpd_port']) # Make sure that no duplicate IDs exist, and that the template exists as well. ids = set() for graph in c['graphs']: graph.setdefault('config', {}) graph['config'].setdefault('periods', []) assert graph['id'] not in ids ids.add(graph['id']) assert(template_exists(graph['template'])) def create_databases(self, overwrite = False): """ A convenience function to create all rrd databases. """ self.validate_config() self.template_runner.create_databases(overwrite) def start_threadpool(self, pool): """ Schedules the start of a threadpool, and schedule the stop of it when the reactor shuts down. """ if not pool.started: reactor.callWhenRunning(self._really_start_threadpool, pool) def _really_start_threadpool(self, pool): """ Starts the threadpool with out scheduleing it via the reactor. """ if pool.started: return pool.start() reactor.addSystemEventTrigger('after', 'shutdown', pool.stop) log.msg('Started threadpool [%s, min=%i, max=%i]' % (pool.name, pool.min, pool.max), logLevel = logging.INFO) def run(self, **config_args): """ Run the stats application, example below: >>> from stats import Stats >>> s = Stats(__name__) >>> s.config.load('config.py') >>> s.create_databases() >>> s.run() """ # Load and validate the configuration. self.config.update(config_args) self.validate_config() # Schedule the start of the threadpools. self.wsgi_threadpool.adjustPoolsize(minthreads = self.config['wsgi_min_threads'], maxthreads=self.config['wsgi_max_threads']) self.start_threadpool(self.wsgi_threadpool) # Start the web server and the template runner. self.template_runner.run() self.flask_app.run() # Finally, start the twisted reactor. reactor.run()