class ClockWithThreads(Clock):
"""
A testing reactor that supplies L{IReactorTime} and L{IReactorThreads}.
"""
def __init__(self):
super(ClockWithThreads, self).__init__()
self._pool = ThreadPool()
def getThreadPool(self):
"""
Get the threadpool.
"""
return self._pool
def suggestThreadPoolSize(self, size):
"""
Approximate the behavior of a "real" reactor.
"""
self._pool.adjustPoolsize(maxthreads=size)
def callInThread(self, thunk, *a, **kw):
"""
No implementation.
"""
def callFromThread(self, thunk, *a, **kw):
"""
class ClockWithThreads(Clock):
"""
A testing reactor that supplies L{IReactorTime} and L{IReactorThreads}.
"""
def __init__(self):
super(ClockWithThreads, self).__init__()
self._pool = ThreadPool()
def getThreadPool(self):
"""
Get the threadpool.
"""
return self._pool
def suggestThreadPoolSize(self, size):
"""
Approximate the behavior of a 'real' reactor.
"""
self._pool.adjustPoolsize(maxthreads=size)
def callInThread(self, thunk, *a, **kw):
"""
No implementation.
"""
def callFromThread(self, thunk, *a, **kw):
"""
def __init__(
self, application, port=80, resources=None, services=None,
loud=False):
service.MultiService.__init__(self)
# Create, start and add a thread pool service, which is made available
# to our WSGIResource within HendrixResource
threads = ThreadPool(name="Hendrix Service")
# Testing threads 1-2-3
threads.adjustPoolsize(3, 5)
reactor.addSystemEventTrigger('after', 'shutdown', threads.stop)
ThreadPoolService(threads).setServiceParent(self)
# create the base resource and add any additional static resources
resource = HendrixResource(reactor, threads, application, loud=loud)
if resources:
resources = sorted(resources, key=lambda r: r.namespace)
for res in resources:
if hasattr(res, 'get_resources'):
for sub_res in res.get_resources():
resource.putNamedChild(sub_res)
else:
resource.putNamedChild(res)
factory = server.Site(resource)
# add a tcp server that binds to port=port
main_web_tcp = TCPServer(port, factory)
main_web_tcp.setName('main_web_tcp')
# to get this at runtime use
# hedrix_service.getServiceNamed('main_web_tcp')
main_web_tcp.setServiceParent(self)
# add any additional services
if services:
for srv_name, srv in services:
srv.setName(srv_name)
srv.setServiceParent(self)
class DirectoryService(BaseDirectoryService):
"""
LDAP directory service.
"""
log = Logger()
fieldName = ConstantsContainer((BaseFieldName, FieldName))
recordType = ConstantsContainer((
BaseRecordType.user,
BaseRecordType.group,
))
def __init__(
self,
url,
baseDN,
credentials=None,
timeout=None,
tlsCACertificateFile=None,
tlsCACertificateDirectory=None,
useTLS=False,
fieldNameToAttributesMap=DEFAULT_FIELDNAME_ATTRIBUTE_MAP,
recordTypeSchemas=DEFAULT_RECORDTYPE_SCHEMAS,
extraFilters=None,
ownThreadpool=True,
threadPoolMax=10,
authConnectionMax=5,
queryConnectionMax=5,
tries=3,
warningThresholdSeconds=5,
_debug=False,
):
"""
@param url: The URL of the LDAP server to connect to.
@type url: L{unicode}
@param baseDN: The base DN for queries.
@type baseDN: L{unicode}
@param credentials: The credentials to use to authenticate with the
LDAP server.
@type credentials: L{IUsernamePassword}
@param timeout: A timeout, in seconds, for LDAP queries.
@type timeout: number
@param tlsCACertificateFile: ...
@type tlsCACertificateFile: L{FilePath}
@param tlsCACertificateDirectory: ...
@type tlsCACertificateDirectory: L{FilePath}
@param useTLS: Enable the use of TLS.
@type useTLS: L{bool}
@param fieldNameToAttributesMap: A mapping of field names to LDAP
attribute names.
@type fieldNameToAttributesMap: mapping with L{NamedConstant} keys and
sequence of L{unicode} values
@param recordTypeSchemas: Schema information for record types.
@type recordTypeSchemas: mapping from L{NamedConstant} to
L{RecordTypeSchema}
@param extraFilters: A dict (keyed off recordType) of extra filter
fragments to AND in to any generated queries.
@type extraFilters: L{dicts} of L{unicode}
"""
self.url = url
self._baseDN = baseDN
self._credentials = credentials
self._timeout = timeout
self._extraFilters = extraFilters
self._tries = tries
self._warningThresholdSeconds = warningThresholdSeconds
if tlsCACertificateFile is None:
self._tlsCACertificateFile = None
else:
self._tlsCACertificateFile = tlsCACertificateFile.path
if tlsCACertificateDirectory is None:
self._tlsCACertificateDirectory = None
else:
self._tlsCACertificateDirectory = tlsCACertificateDirectory.path
self._useTLS = useTLS
if _debug:
self._debug = 255
else:
self._debug = None
if self.fieldName.recordType in fieldNameToAttributesMap:
raise TypeError("Record type field may not be mapped")
if BaseFieldName.uid not in fieldNameToAttributesMap:
raise DirectoryConfigurationError("Mapping for uid required")
self._fieldNameToAttributesMap = fieldNameToAttributesMap
self._attributeToFieldNameMap = {}
for name, attributes in fieldNameToAttributesMap.iteritems():
for attribute in attributes:
if ":" in attribute:
attribute, ignored = attribute.split(":", 1)
self._attributeToFieldNameMap.setdefault(attribute,
[]).append(name)
self._recordTypeSchemas = recordTypeSchemas
attributesToFetch = set()
for attributes in fieldNameToAttributesMap.values():
for attribute in attributes:
if ":" in attribute:
attribute, ignored = attribute.split(":", 1)
attributesToFetch.add(attribute.encode("utf-8"))
self._attributesToFetch = list(attributesToFetch)
# Threaded connection pool.
# The connection size limit here is the size for connections doing
# queries.
# There will also be one-off connections for authentications which also
# run in their own threads.
# Thus the threadpool max ought to be larger than the connection max to
# allow for both pooled query connections and one-off auth-only
# connections.
self.ownThreadpool = ownThreadpool
if self.ownThreadpool:
self.threadpool = ThreadPool(
minthreads=1,
maxthreads=threadPoolMax,
name="LDAPDirectoryService",
)
else:
# Use the default threadpool but adjust its size to fit our needs
self.threadpool = reactor.getThreadPool()
self.threadpool.adjustPoolsize(
max(threadPoolMax, self.threadpool.max))
# Separate pools for LDAP queries and LDAP binds.
self.connectionPools = {
"query": ConnectionPool("query", self, credentials,
queryConnectionMax),
"auth": ConnectionPool("auth", self, None, authConnectionMax),
}
self.poolStats = collections.defaultdict(int)
reactor.callWhenRunning(self.start)
reactor.addSystemEventTrigger("during", "shutdown", self.stop)
def getPreferredRecordTypesOrder(self):
# Not doing this in init( ) because we get our recordTypes assigned later
if not hasattr(self, "_preferredRecordTypesOrder"):
self._preferredRecordTypesOrder = []
for recordTypeName in [
"user", "location", "resource", "group", "address"
]:
try:
recordType = self.recordType.lookupByName(recordTypeName)
self._preferredRecordTypesOrder.append(recordType)
except ValueError:
pass
return self._preferredRecordTypesOrder
def start(self):
"""
Start up this service. Initialize the threadpool (if we own it).
"""
if self.ownThreadpool:
self.threadpool.start()
def stop(self):
"""
Stop the service.
Stop the threadpool if we own it and do other clean-up.
"""
if self.ownThreadpool:
self.threadpool.stop()
# FIXME: we should probably also close the pool of active connections
# too.
@property
def realmName(self):
return u"{self.url}".format(self=self)
class Connection(object):
"""
ContextManager object for getting a connection from the pool.
On exit the connection will be put back in the pool if no exception was
raised.
Otherwise, the connection will be removed from the active connection
list, which will allow a new "clean" connection to be created later if
needed.
"""
def __init__(self, ds, poolName):
self.pool = ds.connectionPools[poolName]
def __enter__(self):
self.connection = self.pool.getConnection()
return self.connection
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self.pool.returnConnection(self.connection)
return True
else:
self.pool.failedConnection(self.connection)
return False
def _authenticateUsernamePassword(self, dn, password):
"""
Open a secondary connection to the LDAP server and try binding to it
with the given credentials
@returns: True if the password is correct, False otherwise
@rtype: deferred C{bool}
@raises: L{LDAPConnectionError} if unable to connect.
"""
d = deferToThreadPool(reactor, self.threadpool,
self._authenticateUsernamePassword_inThread, dn,
password)
qsize = self.threadpool._queue.qsize()
if qsize > 0:
self.log.error("LDAP thread pool overflowing: {qsize}",
qsize=qsize)
self.poolStats["connection-thread-blocked"] += 1
return d
def _authenticateUsernamePassword_inThread(self,
dn,
password,
testStats=None):
"""
Open a secondary connection to the LDAP server and try binding to it
with the given credentials.
This method is always called in a thread.
@returns: True if the password is correct, False otherwise
@rtype: C{bool}
@raises: L{LDAPConnectionError} if unable to connect.
"""
self.log.debug("Authenticating {dn}", dn=dn)
# Retry if we get ldap.SERVER_DOWN
for retryNumber in xrange(self._tries):
# For unit tests, a bit of instrumentation so we can examine
# retryNumber:
if testStats is not None:
testStats["retryNumber"] = retryNumber
try:
with DirectoryService.Connection(self, "auth") as connection:
try:
# During testing, allow an exception to be raised.
# Note: I tried to use patch( ) to accomplish this
# but that seemed to create a race condition in the
# restoration of the patched value and that would cause
# unit tests to occasionally fail.
if testStats is not None:
if "raise" in testStats:
raise testStats["raise"]
connection.simple_bind_s(dn, password)
self.log.debug("Authenticated {dn}", dn=dn)
return True
except (
ldap.INAPPROPRIATE_AUTH,
ldap.INVALID_CREDENTIALS,
ldap.INVALID_DN_SYNTAX,
):
self.log.debug("Unable to authenticate {dn}", dn=dn)
return False
except ldap.CONSTRAINT_VIOLATION:
self.log.info("Account locked {dn}", dn=dn)
return False
except ldap.SERVER_DOWN as e:
# Catch this below for retry
raise e
except Exception as e:
self.log.error(
"Unexpected error {error} trying to authenticate {dn}",
error=str(e),
dn=dn)
return False
else:
# Do an unauthenticated bind on this connection at the end in
# case the server limits the number of concurrent auths by a given user.
connection.simple_bind_s("", "")
except ldap.SERVER_DOWN as e:
self.log.error("LDAP server unavailable")
if retryNumber + 1 == self._tries:
# We've hit SERVER_DOWN self._tries times, giving up.
raise LDAPQueryError("LDAP server down", e)
else:
self.log.error("LDAP connection failure; retrying...")
def _recordsFromQueryString(self,
queryString,
recordTypes=None,
limitResults=None,
timeoutSeconds=None):
d = deferToThreadPool(reactor,
self.threadpool,
self._recordsFromQueryString_inThread,
queryString,
recordTypes,
limitResults=limitResults,
timeoutSeconds=timeoutSeconds)
qsize = self.threadpool._queue.qsize()
if qsize > 0:
self.log.error("LDAP thread pool overflowing: {qsize}",
qsize=qsize)
self.poolStats["connection-thread-blocked"] += 1
return d
def _addExtraFilter(self, recordType, queryString):
if self._extraFilters and self._extraFilters.get(recordType, ""):
queryString = u"(&{extra}{query})".format(
extra=self._extraFilters[recordType], query=queryString)
return queryString
def _recordsFromQueryString_inThread(self,
queryString,
recordTypes=None,
limitResults=None,
timeoutSeconds=None,
testStats=None):
# This method is always called in a thread.
if recordTypes is None:
# recordTypes = list(self.recordTypes())
# Quick hack to optimize the order in which we query by record type:
recordTypes = self.getPreferredRecordTypesOrder()
# Retry if we get ldap.SERVER_DOWN
for retryNumber in xrange(self._tries):
# For unit tests, a bit of instrumentation so we can examine
# retryNumber:
if testStats is not None:
testStats["retryNumber"] = retryNumber
records = []
try:
with DirectoryService.Connection(self, "query") as connection:
for recordType in recordTypes:
if limitResults is not None:
if limitResults < 1:
break
try:
rdn = self._recordTypeSchemas[
recordType].relativeDN
except KeyError:
# Skip this unknown record type
continue
rdn = (ldap.dn.str2dn(rdn.lower()) +
ldap.dn.str2dn(self._baseDN.lower()))
filteredQuery = self._addExtraFilter(
recordType, queryString)
self.log.debug(
"Performing LDAP query: "
"{rdn} {query} {recordType}{limit}{timeout}",
rdn=rdn,
query=filteredQuery.encode("utf-8"),
recordType=recordType,
limit=(" limit={}".format(limitResults)
if limitResults else ""),
timeout=(" timeout={}".format(timeoutSeconds)
if timeoutSeconds else ""),
)
try:
startTime = time.time()
s = ldap. async .List(connection)
s.startSearch(
ldap.dn.dn2str(rdn),
ldap.SCOPE_SUBTREE,
filteredQuery.encode("utf-8"),
attrList=self._attributesToFetch,
timeout=(timeoutSeconds
if timeoutSeconds else -1),
sizelimit=(limitResults
if limitResults else 0),
)
s.processResults()
except ldap.SIZELIMIT_EXCEEDED as e:
self.log.debug(
"LDAP result limit exceeded: {limit}",
limit=limitResults,
)
except ldap.TIMELIMIT_EXCEEDED as e:
self.log.warn(
"LDAP timeout exceeded: {timeout} seconds",
timeout=timeoutSeconds,
)
except ldap.FILTER_ERROR as e:
self.log.error(
"Unable to perform query {query!r}: {err}",
query=queryString,
err=e)
raise LDAPQueryError("Unable to perform query", e)
except ldap.NO_SUCH_OBJECT as e:
# self.log.warn(
# "RDN {rdn} does not exist, skipping", rdn=rdn
# )
continue
except ldap.INVALID_SYNTAX as e:
self.log.error(
"LDAP invalid syntax {query!r}: {err}",
query=queryString,
err=e)
continue
except ldap.SERVER_DOWN as e:
# Catch this below for retry
raise e
except Exception as e:
self.log.error("LDAP error {query!r}: {err}",
query=queryString,
err=e)
raise LDAPQueryError("Unable to perform query", e)
reply = [
resultItem
for _ignore_resultType, resultItem in s.allResults
]
totalTime = time.time() - startTime
if totalTime > self._warningThresholdSeconds:
if filteredQuery and len(filteredQuery) > 500:
filteredQuery = "%s..." % (
filteredQuery[:500], )
self.log.error(
"LDAP query exceeded threshold: {totalTime:.2f} seconds for {rdn} {query} (#results={resultCount})",
totalTime=totalTime,
rdn=rdn,
query=filteredQuery,
resultCount=len(reply))
newRecords = self._recordsFromReply(
reply, recordType=recordType)
self.log.debug(
"Records from LDAP query "
"({rdn} {query} {recordType}): {count}",
rdn=rdn,
query=queryString,
recordType=recordType,
count=len(newRecords))
if limitResults is not None:
limitResults = limitResults - len(newRecords)
records.extend(newRecords)
except ldap.SERVER_DOWN as e:
self.log.error("LDAP server unavailable")
if retryNumber + 1 == self._tries:
# We've hit SERVER_DOWN self._tries times, giving up.
raise LDAPQueryError("LDAP server down", e)
else:
self.log.error("LDAP connection failure; retrying...")
else:
# Only retry if we got ldap.SERVER_DOWN, otherwise break out of
# loop.
break
self.log.debug("LDAP result count ({query}): {count}",
query=queryString,
count=len(records))
return records
def _recordWithDN(self, dn):
d = deferToThreadPool(reactor, self.threadpool,
self._recordWithDN_inThread, dn)
qsize = self.threadpool._queue.qsize()
if qsize > 0:
self.log.error("LDAP thread pool overflowing: {qsize}",
qsize=qsize)
self.poolStats["connection-thread-blocked"] += 1
return d
def _recordWithDN_inThread(self, dn, testStats=None):
"""
@param dn: The DN of the record to search for
@type dn: C{str}
"""
# This method is always called in a thread.
records = []
# Retry if we get ldap.SERVER_DOWN
for retryNumber in xrange(self._tries):
# For unit tests, a bit of instrumentation:
if testStats is not None:
testStats["retryNumber"] = retryNumber
try:
with DirectoryService.Connection(self, "query") as connection:
self.log.debug("Performing LDAP DN query: {dn}", dn=dn)
try:
reply = connection.search_s(
dn,
ldap.SCOPE_SUBTREE,
"(objectClass=*)",
attrlist=self._attributesToFetch)
records = self._recordsFromReply(reply)
except ldap.NO_SUCH_OBJECT:
records = []
except ldap.INVALID_DN_SYNTAX:
self.log.warn("Invalid LDAP DN syntax: '{dn}'", dn=dn)
records = []
except ldap.SERVER_DOWN as e:
self.log.error("LDAP server unavailable")
if retryNumber + 1 == self._tries:
# We've hit SERVER_DOWN self._tries times, giving up
raise LDAPQueryError("LDAP server down", e)
else:
self.log.error("LDAP connection failure; retrying...")
else:
# Only retry if we got ldap.SERVER_DOWN, otherwise break out of
# loop
break
if len(records):
return records[0]
else:
return None
def _recordsFromReply(self, reply, recordType=None):
records = []
for dn, recordData in reply:
# Determine the record type
if recordType is None:
recordType = recordTypeForDN(self._baseDN,
self._recordTypeSchemas, dn)
if recordType is None:
recordType = recordTypeForRecordData(self._recordTypeSchemas,
recordData)
if recordType is None:
self.log.debug(
"Ignoring LDAP record data; unable to determine record "
"type: {recordData!r}",
recordData=recordData,
)
continue
# Populate a fields dictionary
fields = {}
for fieldName, attributeRules in (
self._fieldNameToAttributesMap.iteritems()):
valueType = self.fieldName.valueType(fieldName)
for attributeRule in attributeRules:
attributeName = attributeRule.split(":")[0]
if attributeName in recordData:
values = recordData[attributeName]
if valueType in (unicode, UUID):
if not isinstance(values, list):
values = [values]
if valueType is unicode:
newValues = []
for v in values:
if isinstance(v, unicode):
# because the ldap unit test produces
# unicode values (?)
newValues.append(v)
else:
try:
newValues.append(
unicode(v, "utf-8"))
except UnicodeDecodeError:
# Log and re-raise so the net behavior is as before during debugging
self.log.error(
"Received non-UTF-8 bytes from LDAP for {dn} in {name}",
dn=dn,
name=fieldName)
raise
else:
try:
newValues = [valueType(v) for v in values]
except Exception, e:
self.log.warn(
"Can't parse value {name} {values} "
"({error})",
name=fieldName,
values=values,
error=str(e))
continue
if self.fieldName.isMultiValue(fieldName):
if fieldName in fields:
fields[fieldName].extend(newValues)
else:
fields[fieldName] = newValues
else:
# First one in the list wins
if fieldName not in fields:
fields[fieldName] = newValues[0]
elif valueType is bool:
if not isinstance(values, list):
values = [values]
if ":" in attributeRule:
ignored, trueValue = attributeRule.split(":")
else:
trueValue = "true"
for value in values:
if value == trueValue:
fields[fieldName] = True
break
else:
fields[fieldName] = False
elif issubclass(valueType, Names):
if not isinstance(values, list):
values = [values]
_ignore_attribute, attributeValue, fieldValue = (
attributeRule.split(":"))
for value in values:
if value == attributeValue:
# convert to a constant
try:
fieldValue = (
valueType.lookupByName(fieldValue))
fields[fieldName] = fieldValue
except ValueError:
pass
break
else:
raise LDAPConfigurationError(
"Unknown value type {0} for field {1}".format(
valueType, fieldName))
# Skip any results missing the uid, which is a required field
if self.fieldName.uid not in fields:
continue
# Set record type and dn fields
fields[self.fieldName.recordType] = recordType
fields[self.fieldName.dn] = dn.decode("utf-8")
# Make a record object from fields.
record = DirectoryRecord(self, fields)
records.append(record)
# self.log.debug("LDAP results: {records}", records=records)
return records
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from protocol import format_string
from twisted.python.threadpool import ThreadPool
import logging
import time
logger = logging.getLogger(name='plugin')
pool = ThreadPool()
pool.adjustPoolsize(1)
pool.start()
class Plugin:
"""
Base class of a plugin. Use this when writing new plugins. Provides
several methods to communicate with the clients.
Provide at least these methods:
- name(): String ID of the plugin. It must match the name in the client
counterpart.
- message_from_client(message): Called when the client sends some data,
usually as a response to some request.
"""
def __init__(self, plugins):
class DirectoryService(BaseDirectoryService):
"""
LDAP directory service.
"""
log = Logger()
fieldName = ConstantsContainer((BaseFieldName, FieldName))
recordType = ConstantsContainer((
BaseRecordType.user,
BaseRecordType.group,
))
def __init__(
self,
url,
baseDN,
credentials=None,
timeout=None,
tlsCACertificateFile=None,
tlsCACertificateDirectory=None,
useTLS=False,
fieldNameToAttributesMap=DEFAULT_FIELDNAME_ATTRIBUTE_MAP,
recordTypeSchemas=DEFAULT_RECORDTYPE_SCHEMAS,
extraFilters=None,
ownThreadpool=True,
threadPoolMax=10,
connectionMax=10,
tries=3,
_debug=False,
):
"""
@param url: The URL of the LDAP server to connect to.
@type url: L{unicode}
@param baseDN: The base DN for queries.
@type baseDN: L{unicode}
@param credentials: The credentials to use to authenticate with the
LDAP server.
@type credentials: L{IUsernamePassword}
@param timeout: A timeout, in seconds, for LDAP queries.
@type timeout: number
@param tlsCACertificateFile: ...
@type tlsCACertificateFile: L{FilePath}
@param tlsCACertificateDirectory: ...
@type tlsCACertificateDirectory: L{FilePath}
@param useTLS: Enable the use of TLS.
@type useTLS: L{bool}
@param fieldNameToAttributesMap: A mapping of field names to LDAP
attribute names.
@type fieldNameToAttributesMap: mapping with L{NamedConstant} keys and
sequence of L{unicode} values
@param recordTypeSchemas: Schema information for record types.
@type recordTypeSchemas: mapping from L{NamedConstant} to
L{RecordTypeSchema}
@param extraFilters: A dict (keyed off recordType) of extra filter
fragments to AND in to any generated queries.
@type extraFilters: L{dicts} of L{unicode}
"""
self.url = url
self._baseDN = baseDN
self._credentials = credentials
self._timeout = timeout
self._extraFilters = extraFilters
self._tries = tries
if tlsCACertificateFile is None:
self._tlsCACertificateFile = None
else:
self._tlsCACertificateFile = tlsCACertificateFile.path
if tlsCACertificateDirectory is None:
self._tlsCACertificateDirectory = None
else:
self._tlsCACertificateDirectory = tlsCACertificateDirectory.path
self._useTLS = useTLS
if _debug:
self._debug = 255
else:
self._debug = None
if self.fieldName.recordType in fieldNameToAttributesMap:
raise TypeError("Record type field may not be mapped")
if BaseFieldName.uid not in fieldNameToAttributesMap:
raise DirectoryConfigurationError("Mapping for uid required")
self._fieldNameToAttributesMap = fieldNameToAttributesMap
self._attributeToFieldNameMap = {}
for name, attributes in fieldNameToAttributesMap.iteritems():
for attribute in attributes:
if ":" in attribute:
attribute, ignored = attribute.split(":", 1)
self._attributeToFieldNameMap.setdefault(attribute,
[]).append(name)
self._recordTypeSchemas = recordTypeSchemas
attributesToFetch = set()
for attributes in fieldNameToAttributesMap.values():
for attribute in attributes:
if ":" in attribute:
attribute, ignored = attribute.split(":", 1)
attributesToFetch.add(attribute.encode("utf-8"))
self._attributesToFetch = list(attributesToFetch)
# Threaded connection pool. The connection size limit here is the size for connections doing queries.
# There will also be one-off connections for authentications which also run in their own threads. Thus
# the threadpool max ought to be larger than the connection max to allow for both pooled query connections
# and one-off auth-only connections.
self.ownThreadpool = ownThreadpool
if self.ownThreadpool:
self.threadpool = ThreadPool(minthreads=1,
maxthreads=threadPoolMax,
name="LDAPDirectoryService")
else:
# Use the default threadpool but adjust its size to fit our needs
self.threadpool = reactor.getThreadPool()
self.threadpool.adjustPoolsize(
max(threadPoolMax, self.threadpool.max))
self.connectionMax = connectionMax
self.connectionCreateLock = RLock()
self.connections = []
self.connectionQueue = Queue()
self.poolStats = collections.defaultdict(int)
self.activeCount = 0
reactor.callWhenRunning(self.start)
reactor.addSystemEventTrigger('during', 'shutdown', self.stop)
def start(self):
"""
Start up this service. Initialize the threadpool (if we own it).
"""
if self.ownThreadpool:
self.threadpool.start()
def stop(self):
"""
Stop the service. Stop the threadpool if we own it and do other clean-up.
"""
if self.ownThreadpool:
self.threadpool.stop()
# FIXME: we should probably also close the pool of active connections too
@property
def realmName(self):
return u"{self.url}".format(self=self)
class Connection(object):
"""
ContextManager object for getting a connection from the pool. On exit the connection
will be put back in the pool if no exception was raised. Otherwise, the connection will be
removed from the active connection list, which will allow a new "clean" connection to
be created later if needed.
"""
def __init__(self, ds):
self.ds = ds
def __enter__(self):
self.connection = self.ds._getConnection()
return self.connection
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self.ds._returnConnection(self.connection)
return True
else:
self.ds._failedConnection(self.connection)
return False
def _getConnection(self):
"""
Get a connection from the connection pool. This will retrieve a connection from the connection
pool L{Queue} object. If the L{Queue} is empty, it will check to see whether a new connection can
be created (based on the connection limit), and if so create that and use it. If no new
connections can be created, it will block on the L{Queue} until an existing, in-use, connection
is put back.
"""
try:
connection = self.connectionQueue.get(block=False)
except Empty:
# Note we use a lock here to prevent a race condition in which multiple requests for a new connection
# could succeed even though the connection counts starts out one less than the maximum. This can happen
# because self._connect() can take a while.
self.connectionCreateLock.acquire()
if len(self.connections) < self.connectionMax:
connection = self._connect()
self.connections.append(connection)
self.connectionCreateLock.release()
else:
self.connectionCreateLock.release()
self.poolStats["connection-blocked"] += 1
connection = self.connectionQueue.get()
self.poolStats["connection-{}".format(
self.connections.index(connection))] += 1
self.activeCount += 1
self.poolStats["connection-max"] = max(
self.poolStats["connection-max"], self.activeCount)
return connection
def _returnConnection(self, connection):
"""
A connection is no longer needed - return it to the pool.
"""
self.activeCount -= 1
self.connectionQueue.put(connection)
def _failedConnection(self, connection):
"""
A connection has failed - remove it from the list of active connections. A new
one will be created if needed.
"""
self.activeCount -= 1
self.poolStats["connection-errors"] += 1
self.connections.remove(connection)
def _connect(self):
"""
Connect to the directory server.
This will always be called in a thread to prevent blocking.
@returns: The connection object.
@rtype: L{ldap.ldapobject.LDAPObject}
@raises: L{LDAPConnectionError} if unable to connect.
"""
# FIXME: ldap connection objects are not thread safe, so let's set up
# a connection pool
self.log.debug("Connecting to LDAP at {log_source.url}")
connection = self._newConnection()
if self._credentials is not None:
if IUsernamePassword.providedBy(self._credentials):
try:
connection.simple_bind_s(
self._credentials.username,
self._credentials.password,
)
self.log.debug("Bound to LDAP as {credentials.username}",
credentials=self._credentials)
except (ldap.INVALID_CREDENTIALS, ldap.INVALID_DN_SYNTAX) as e:
self.log.error(
"Unable to bind to LDAP as {credentials.username}",
credentials=self._credentials)
raise LDAPBindAuthError(self._credentials.username, e)
else:
raise LDAPConnectionError(
"Unknown credentials type: {0}".format(self._credentials))
return connection
def _newConnection(self):
"""
Create a new LDAP connection and initialize and start TLS if required.
This will always be called in a thread to prevent blocking.
@returns: The connection object.
@rtype: L{ldap.ldapobject.LDAPObject}
@raises: L{LDAPConnectionError} if unable to connect.
"""
connection = ldap.initialize(self.url)
# FIXME: Use trace_file option to wire up debug logging when
# Twisted adopts the new logging stuff.
for option, value in (
(ldap.OPT_TIMEOUT, self._timeout),
(ldap.OPT_X_TLS_CACERTFILE, self._tlsCACertificateFile),
(ldap.OPT_X_TLS_CACERTDIR, self._tlsCACertificateDirectory),
(ldap.OPT_DEBUG_LEVEL, self._debug),
):
if value is not None:
connection.set_option(option, value)
if self._useTLS:
self.log.debug("Starting TLS for {log_source.url}")
connection.start_tls_s()
return connection
def _authenticateUsernamePassword(self, dn, password):
"""
Open a secondary connection to the LDAP server and try binding to it
with the given credentials
@returns: True if the password is correct, False otherwise
@rtype: deferred C{bool}
@raises: L{LDAPConnectionError} if unable to connect.
"""
return deferToThreadPool(reactor, self.threadpool,
self._authenticateUsernamePassword_inThread,
dn, password)
def _authenticateUsernamePassword_inThread(self, dn, password):
"""
Open a secondary connection to the LDAP server and try binding to it
with the given credentials.
This method is always called in a thread.
@returns: True if the password is correct, False otherwise
@rtype: C{bool}
@raises: L{LDAPConnectionError} if unable to connect.
"""
self.log.debug("Authenticating {dn}", dn=dn)
connection = self._newConnection()
try:
connection.simple_bind_s(dn, password)
self.log.debug("Authenticated {dn}", dn=dn)
return True
except (ldap.INVALID_CREDENTIALS, ldap.INVALID_DN_SYNTAX):
self.log.debug("Unable to authenticate {dn}", dn=dn)
return False
finally:
# TODO: should we explicitly "close" the connection in a finally
# clause rather than just let it go out of scope and be garbage collected
# at some indeterminate point in the future? Up side is that we won't hang
# on to the connection or other resources for longer than needed. Down side
# is we will take a little bit of extra time in this call to close it down.
# If we do decide to "close" then we probably have to use one of the "unbind"
# methods on the L{LDAPObject}.
connection = None
def _recordsFromQueryString(self,
queryString,
recordTypes=None,
limitResults=None,
timeoutSeconds=None):
return deferToThreadPool(reactor,
self.threadpool,
self._recordsFromQueryString_inThread,
queryString,
recordTypes,
limitResults=limitResults,
timeoutSeconds=timeoutSeconds)
def _addExtraFilter(self, recordType, queryString):
if self._extraFilters and self._extraFilters.get(recordType, ""):
queryString = "(&{extra}{query})".format(
extra=self._extraFilters[recordType], query=queryString)
return queryString
def _recordsFromQueryString_inThread(self,
queryString,
recordTypes=None,
limitResults=None,
timeoutSeconds=None):
"""
This method is always called in a thread.
"""
if recordTypes is None:
recordTypes = list(self.recordTypes())
# Retry if we get ldap.SERVER_DOWN
for self._retryNumber in xrange(self._tries):
records = []
try:
with DirectoryService.Connection(self) as connection:
for recordType in recordTypes:
if limitResults is not None:
if limitResults < 1:
break
try:
rdn = self._recordTypeSchemas[
recordType].relativeDN
except KeyError:
# Skip this unknown record type
continue
rdn = (ldap.dn.str2dn(rdn.lower()) +
ldap.dn.str2dn(self._baseDN.lower()))
filteredQuery = self._addExtraFilter(
recordType, queryString)
self.log.debug(
"Performing LDAP query: {rdn} {query} {recordType}{limit}{timeout}",
rdn=rdn,
query=filteredQuery,
recordType=recordType,
limit=" limit={}".format(limitResults)
if limitResults else "",
timeout=" timeout={}".format(timeoutSeconds)
if timeoutSeconds else "",
)
try:
s = ldap. async .List(connection)
s.startSearch(
ldap.dn.dn2str(rdn),
ldap.SCOPE_SUBTREE,
filteredQuery,
attrList=self._attributesToFetch,
timeout=timeoutSeconds
if timeoutSeconds else -1,
sizelimit=limitResults if limitResults else 0)
s.processResults()
except ldap.SIZELIMIT_EXCEEDED as e:
self.log.debug(
"LDAP result limit exceeded: {}".format(
limitResults, ))
except ldap.TIMELIMIT_EXCEEDED as e:
self.log.warn(
"LDAP timeout exceeded: {} seconds".format(
timeoutSeconds, ))
except ldap.FILTER_ERROR as e:
self.log.error(
"Unable to perform query {query!r}: {err}",
query=queryString,
err=e)
raise LDAPQueryError("Unable to perform query", e)
except ldap.NO_SUCH_OBJECT as e:
# self.log.warn("RDN {rdn} does not exist, skipping", rdn=rdn)
continue
except ldap.INVALID_SYNTAX as e:
self.log.error(
"LDAP invalid syntax {query!r}: {err}",
query=queryString,
err=e)
continue
except ldap.SERVER_DOWN as e:
# Catch this below for retry
raise e
except Exception as e:
self.log.error("LDAP error {query!r}: {err}",
query=queryString,
err=e)
raise LDAPQueryError("Unable to perform query", e)
reply = [
resultItem
for _ignore_resultType, resultItem in s.allResults
]
newRecords = self._recordsFromReply(
reply, recordType=recordType)
self.log.debug(
"Records from LDAP query ({rdn} {query} {recordType}): {count}",
rdn=rdn,
query=queryString,
recordType=recordType,
count=len(newRecords))
if limitResults is not None:
limitResults = limitResults - len(newRecords)
records.extend(newRecords)
except ldap.SERVER_DOWN as e:
self.log.error("LDAP server unavailable")
if self._retryNumber + 1 == self._tries:
# We've hit SERVER_DOWN self._tries times, giving up
raise LDAPQueryError("LDAP server down", e)
else:
self.log.error("LDAP connection failure; retrying...")
else:
# Only retry if we got ldap.SERVER_DOWN, otherwise break out of
# loop
break
self.log.debug("LDAP result count ({query}): {count}",
query=queryString,
count=len(records))
return records
def _recordWithDN(self, dn):
return deferToThreadPool(reactor, self.threadpool,
self._recordWithDN_inThread, dn)
def _recordWithDN_inThread(self, dn):
"""
This method is always called in a thread.
@param dn: The DN of the record to search for
@type dn: C{str}
"""
records = []
# Retry if we get ldap.SERVER_DOWN
for self._retryNumber in xrange(self._tries):
try:
with DirectoryService.Connection(self) as connection:
self.log.debug("Performing LDAP DN query: {dn}", dn=dn)
try:
reply = connection.search_s(
dn,
ldap.SCOPE_SUBTREE,
"(objectClass=*)",
attrlist=self._attributesToFetch)
records = self._recordsFromReply(reply)
except ldap.NO_SUCH_OBJECT:
records = []
except ldap.INVALID_DN_SYNTAX:
self.log.warn("Invalid LDAP DN syntax: '{dn}'", dn=dn)
records = []
except ldap.SERVER_DOWN as e:
self.log.error("LDAP server unavailable")
if self._retryNumber + 1 == self._tries:
# We've hit SERVER_DOWN self._tries times, giving up
raise LDAPQueryError("LDAP server down", e)
else:
self.log.error("LDAP connection failure; retrying...")
else:
# Only retry if we got ldap.SERVER_DOWN, otherwise break out of
# loop
break
if len(records):
return records[0]
else:
return None
def _recordsFromReply(self, reply, recordType=None):
records = []
for dn, recordData in reply:
# Determine the record type
if recordType is None:
recordType = recordTypeForDN(self._baseDN,
self._recordTypeSchemas, dn)
if recordType is None:
recordType = recordTypeForRecordData(self._recordTypeSchemas,
recordData)
if recordType is None:
self.log.debug(
"Ignoring LDAP record data; unable to determine record "
"type: {recordData!r}",
recordData=recordData,
)
continue
# Populate a fields dictionary
fields = {}
for fieldName, attributeRules in self._fieldNameToAttributesMap.iteritems(
):
valueType = self.fieldName.valueType(fieldName)
for attributeRule in attributeRules:
attributeName = attributeRule.split(":")[0]
if attributeName in recordData:
values = recordData[attributeName]
if valueType in (unicode, UUID):
if not isinstance(values, list):
values = [values]
if valueType is unicode:
newValues = []
for v in values:
if isinstance(v, unicode):
# because the ldap unit test produces
# unicode values (?)
newValues.append(v)
else:
newValues.append(unicode(v, "utf-8"))
else:
try:
newValues = [valueType(v) for v in values]
except Exception, e:
self.log.warn(
"Can't parse value {name} {values} ({error})",
name=fieldName,
values=values,
error=str(e))
continue
if self.fieldName.isMultiValue(fieldName):
if fieldName in fields:
fields[fieldName].extend(newValues)
else:
fields[fieldName] = newValues
else:
# First one in the list wins
if fieldName not in fields:
fields[fieldName] = newValues[0]
elif valueType is bool:
if not isinstance(values, list):
values = [values]
if ":" in attributeRule:
ignored, trueValue = attributeRule.split(":")
else:
trueValue = "true"
for value in values:
if value == trueValue:
fields[fieldName] = True
break
else:
fields[fieldName] = False
elif issubclass(valueType, Names):
if not isinstance(values, list):
values = [values]
_ignore_attribute, attributeValue, fieldValue = attributeRule.split(
":")
for value in values:
if value == attributeValue:
# convert to a constant
try:
fieldValue = valueType.lookupByName(
fieldValue)
fields[fieldName] = fieldValue
except ValueError:
pass
break
else:
raise LDAPConfigurationError(
"Unknown value type {0} for field {1}".format(
valueType, fieldName))
# Skip any results missing the uid, which is a required field
if self.fieldName.uid not in fields:
continue
# Set record type and dn fields
fields[self.fieldName.recordType] = recordType
fields[self.fieldName.dn] = dn.decode("utf-8")
# Make a record object from fields.
record = DirectoryRecord(self, fields)
records.append(record)
# self.log.debug("LDAP results: {records}", records=records)
return records
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # from protocol import format_string from twisted.python.threadpool import ThreadPool import logging import time logger = logging.getLogger(name='plugin') pool = ThreadPool() pool.adjustPoolsize(1) pool.start() class Plugin: """ Base class of a plugin. Use this when writing new plugins. Provides several methods to communicate with the clients. Provide at least these methods: - name(): String ID of the plugin. It must match the name in the client counterpart. - message_from_client(message): Called when the client sends some data, usually as a response to some request. """ def __init__(self, plugins): """
class Stats(object):
"""
stats.Stats rrdtool templated poller, updater, and web frontend
"""
# Our default configuration directives, if they don't
# exist in the configuration file, they'll reflect what's here
default_config = ImmutableDict(**dict(
graphs = [],
httpd_port = 8080,
interface = '0.0.0.0',
graph_draw_frequency = dict(
hour = 60,
day = 300,
week = 300*3,
default = 60
),
wsgi_min_threads = 1,
wsgi_max_threads = 5
))
def __init__(self, instance_name):
self.instance_name = instance_name
self.config = Config(self.default_config)
self.flask_app = WebApp(self)
self.template_runner = TemplateRunner(self)
self.active_graphs = dict()
self.last_draw_timestamp = dict()
self.wsgi_threadpool = ThreadPool(minthreads = self.config['wsgi_min_threads'], maxthreads=self.config['wsgi_max_threads'], name = 'wsgi_threadpool')
def validate_config(self):
""" Validates the loaded configuration. """
c = self.config
# Make sure that we have a database_path, and an image_path...
assert 'database_path' in c
assert 'image_path' in c
# We should probably check if these paths exist and make them as well...
# Set the default values.
graph_draw_frequency = c['graph_draw_frequency']
for period, interval in self.default_config['graph_draw_frequency'].iteritems():
graph_draw_frequency.setdefault(period, interval)
# A quick check to make sure that our port is an integer.
c['httpd_port'] = int(c['httpd_port'])
# Make sure that no duplicate IDs exist, and that the template exists as well.
ids = set()
for graph in c['graphs']:
graph.setdefault('config', {})
graph['config'].setdefault('periods', [])
assert graph['id'] not in ids
ids.add(graph['id'])
assert(template_exists(graph['template']))
def create_databases(self, overwrite = False):
""" A convenience function to create all rrd databases. """
self.validate_config()
self.template_runner.create_databases(overwrite)
def start_threadpool(self, pool):
""" Schedules the start of a threadpool, and schedule the stop of it when the reactor shuts down. """
if not pool.started:
reactor.callWhenRunning(self._really_start_threadpool, pool)
def _really_start_threadpool(self, pool):
""" Starts the threadpool with out scheduleing it via the reactor. """
if pool.started:
return
pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
log.msg('Started threadpool [%s, min=%i, max=%i]' % (pool.name, pool.min, pool.max), logLevel = logging.INFO)
def run(self, **config_args):
"""
Run the stats application, example below:
>>> from stats import Stats
>>> s = Stats(__name__)
>>> s.config.load('config.py')
>>> s.create_databases()
>>> s.run()
"""
# Load and validate the configuration.
self.config.update(config_args)
self.validate_config()
# Schedule the start of the threadpools.
self.wsgi_threadpool.adjustPoolsize(minthreads = self.config['wsgi_min_threads'], maxthreads=self.config['wsgi_max_threads'])
self.start_threadpool(self.wsgi_threadpool)
# Start the web server and the template runner.
self.template_runner.run()
self.flask_app.run()
# Finally, start the twisted reactor.
reactor.run()
