def test_lookupByName(self): constants = set(( Instruments.hammer, Tools.screwdriver, Instruments.chisel, )) container = ConstantsContainer(constants) self.assertEquals( container.lookupByName("hammer"), Instruments.hammer, ) self.assertEquals( container.lookupByName("screwdriver"), Tools.screwdriver, ) self.assertEquals( container.lookupByName("chisel"), Instruments.chisel, ) self.assertRaises( ValueError, container.lookupByName, "plugh", )
class DirectoryService(BaseDirectoryService): """ XML directory service with calendar and contacts data. """ recordType = ConstantsContainer( (BaseDirectoryService.recordType, RecordType)) # MOVE2WHO: Wilfredo had added augment fields into xml, which does make # some sense, but for backwards compatibility right now I will take those # out, and rely on a separate augment service # fieldName = ConstantsContainer( # (BaseDirectoryService.fieldName, FieldName) # ) # XML schema constants element = ConstantsContainer((BaseDirectoryService.element, Element)) attribute = ConstantsContainer((BaseDirectoryService.attribute, Attribute)) recordTypeValue = ConstantsContainer( (BaseDirectoryService.recordTypeValue, RecordTypeValue)) xmlFieldOrder = BaseDirectoryService.xmlFieldOrder + ( FieldName.capacity, FieldName.floor, FieldName.associatedAddress, FieldName.abbreviatedName, FieldName.streetAddress, FieldName.geographicLocation, )
class DirectoryService(BaseDirectoryService): """ XML directory service with calendar and contacts data. """ recordType = ConstantsContainer( (BaseDirectoryService.recordType, RecordType) ) # MOVE2WHO: Wilfredo had added augment fields into xml, which does make # some sense, but for backwards compatibility right now I will take those # out, and rely on a separate augment service # fieldName = ConstantsContainer( # (BaseDirectoryService.fieldName, FieldName) # ) # XML schema constants element = ConstantsContainer( (BaseDirectoryService.element, Element) ) attribute = ConstantsContainer( (BaseDirectoryService.attribute, Attribute) ) recordTypeValue = ConstantsContainer( (BaseDirectoryService.recordTypeValue, RecordTypeValue) )
def test_iterconstants(self): constants = set((Tools.hammer, Tools.screwdriver, Instruments.chisel)) container = ConstantsContainer(constants) self.assertEquals( set(container.iterconstants()), constants, )
def service(self, **kwargs): svc = TestService( url=self.url, baseDN=self.baseDN, fieldNameToAttributesMap=TEST_FIELDNAME_MAP, recordTypeSchemas=MappingProxyType({ RecordType.user: RecordTypeSchema( relativeDN=u"cn=user", # (objectClass=inetOrgPerson) attributes=(( LDAPAttribute.objectClass.value, LDAPObjectClass.inetOrgPerson.value, ), ), ), RecordType.group: RecordTypeSchema( relativeDN=u"cn=group", # (objectClass=groupOfNames) attributes=(( LDAPAttribute.objectClass.value, LDAPObjectClass.groupOfUniqueNames.value, ), ), ), }), **kwargs) svc.fieldName = ConstantsContainer((svc.fieldName, TestFieldName)) return svc
class InMemoryDirectoryService(IndexDirectoryService): """ An in-memory IDirectoryService. You must call updateRecords( ) if you want to populate this service. """ recordType = ConstantsContainer( (RecordType.user, RecordType.group, CalRecordType.location, CalRecordType.resource, CalRecordType.address)) def loadRecords(self): pass @inlineCallbacks def updateRecords(self, records, create=False): recordsByUID = dict(((record.uid, record) for record in records)) if not create: # Make sure all the records already exist for uid, _ignore_record in recordsByUID.items(): if uid not in self._index[self.fieldName.uid]: raise NoSuchRecordError(uid) yield self.removeRecords(recordsByUID.keys()) self.indexRecords(records) def removeRecords(self, uids): index = self._index for fieldName in self.indexedFields: for recordSet in index[fieldName].itervalues(): for record in list(recordSet): if record.uid in uids: recordSet.remove(record) return succeed(None)
class DirectoryService(BaseDirectoryService): recordType = ConstantsContainer( (BaseDirectoryService.recordType, RecordType)) def __init__(self): BaseDirectoryService.__init__(self, realmName="no realm") def recordWithUID(self, uid, timeoutSeconds=None): return DirectoryRecord(self, uid) def recordWithShortName(self, recordType, shortName, timeoutSeconds=None): return DirectoryRecord(self, shortName)
class DirectoryService(BaseDirectoryService): """ Duty Management System directory service. """ log = Logger() fieldName = ConstantsContainer((BaseFieldName, IndexFieldName, FieldName)) recordType = ConstantsContainer(( BaseRecordType.user, BaseRecordType.group, )) def __init__(self, dms, masterKey=None): BaseDirectoryService.__init__(self, realmName=noRealmName) self.dms = dms self._personnel = None self._positions = None self._masterKey = masterKey @property def realmName(self): return "{}@{}".format(self.dms.database, self.dms.host) @realmName.setter def realmName(self, value): if value is not noRealmName: raise AttributeError("realmName may not be set directly") def loadRecords(self): # Getting the personnel data from DMS is async, and this API is not, # so we're going to call into an async method and when it's eventually # done, we'll have some data, but we have no way to tell the caller # when that is. self._loadRecordsFromPersonnel() @inlineCallbacks def _loadRecordsFromPersonnel(self): try: personnel = yield self.dms.personnel() positions = yield self.dms.positions() except DatabaseError as e: self.log.error("Unable to look up personnel data: {error}", error=e) return if personnel is self._personnel and positions is self._positions: return self.flush() self.indexRecords( RangerDirectoryRecord(self, ranger) for ranger in personnel) self.indexRecords( PositionDirectoryRecord(self, position) for position in positions) self.log.info("DMS directory service updated.") self._personnel = personnel self._positions = positions
class CachingDirectoryService( BaseDirectoryService, CalendarDirectoryServiceMixin ): """ Caching directory service. This is a directory service that wraps an L{IDirectoryService} and caches directory records. """ fieldName = ConstantsContainer(( BaseFieldName, FieldName, )) def __init__(self, directory, expireSeconds=30, lookupsBetweenPurges=0, negativeCaching=True): BaseDirectoryService.__init__(self, directory.realmName) self._directory = directory # Patch the wrapped directory service's recordWithXXX to instead # use this cache directory._wrapped_recordWithUID = directory.recordWithUID directory.recordWithUID = self.recordWithUID directory._wrapped_recordWithGUID = directory.recordWithGUID directory.recordWithGUID = self.recordWithGUID directory._wrapped_recordWithShortName = directory.recordWithShortName directory.recordWithShortName = self.recordWithShortName directory._wrapped_recordsWithEmailAddress = directory.recordsWithEmailAddress directory.recordsWithEmailAddress = self.recordsWithEmailAddress self._expireSeconds = expireSeconds if lookupsBetweenPurges == 0: self._purgingEnabled = False else: self._purgingEnabled = True self._lookupsBetweenPurges = lookupsBetweenPurges self.negativeCaching = negativeCaching self.resetCache() def setTimingMethod(self, f): """ Replace the default no-op timing method """ self._addTiming = f def _addTiming(self, key, duration): """ Timing won't get recorded by default -- you must call setTimingMethod with a callable that takes a key such as a method name, and a duration. """ pass def resetCache(self): """ Clear the cache """ log.debug("Resetting cache") self._cache = { IndexType.uid: {}, IndexType.guid: {}, IndexType.shortName: {}, # key is (recordType.name, shortName) IndexType.emailAddress: {}, } self._negativeCache = { IndexType.uid: {}, IndexType.guid: {}, IndexType.shortName: {}, # key is (recordType.name, shortName) IndexType.emailAddress: {}, } self._hitCount = 0 self._requestCount = 0 if self._purgingEnabled: self._lookupsUntilScan = self._lookupsBetweenPurges # If DPS is in use we restrict the cache to the DPSClients only, otherwise we can # cache in each worker process if config.Memcached.Pools.Default.ClientEnabled and ( not config.DirectoryProxy.Enabled or isinstance(self._directory, DPSClientDirectoryService) ): self._memcacher = DirectoryMemcacher( self._expireSeconds, self._directory, self._directory.realmName, "a" if config.DirectoryProxy.Enabled else "b" ) else: self._memcacher = None def setTestTime(self, timestamp): """ Only used for unit tests to override the notion of "now" @param timestamp: seconds @type timestamp: C{float} """ self._test_time = timestamp def cacheRecord(self, record, indexTypes, addToMemcache=True): """ Store a record in the cache, within the specified indexes @param record: the directory record @param indexTypes: an iterable of L{IndexType} """ if hasattr(self, "_test_time"): timestamp = self._test_time else: timestamp = time.time() cached = [] if IndexType.uid in indexTypes: self._cache[IndexType.uid][record.uid] = (timestamp, record) cached.append((IndexType.uid, record.uid,)) if IndexType.guid in indexTypes: try: self._cache[IndexType.guid][record.guid] = (timestamp, record) cached.append((IndexType.guid, record.guid,)) except AttributeError: pass if IndexType.shortName in indexTypes: try: typeName = record.recordType.name for name in record.shortNames: self._cache[IndexType.shortName][(typeName, name)] = (timestamp, record) cached.append((IndexType.shortName, (typeName, name),)) except AttributeError: pass if IndexType.emailAddress in indexTypes: try: for emailAddress in record.emailAddresses: self._cache[IndexType.emailAddress][emailAddress] = (timestamp, record) cached.append((IndexType.emailAddress, emailAddress,)) except AttributeError: pass if addToMemcache and self._memcacher is not None: for indexType, key in cached: memcachekey = self._memcacher.generateMemcacheKey(indexType, key) log.debug("Memcache: storing %s" % (memcachekey,)) try: self._memcacher.memcacheSetRecord(memcachekey, record) except DirectoryMemcacheError: log.error("Memcache: failed to store %s" % (memcachekey,)) pass def negativeCacheRecord(self, indexType, key): """ Store a record in the negative cache, within the specified indexes @param record: the directory record @param indexType: an L{IndexType} """ if hasattr(self, "_test_time"): timestamp = self._test_time else: timestamp = time.time() self._negativeCache[indexType][key] = timestamp # Do memcache if self._memcacher is not None: # The only time the recordType arg matters is when indexType is # short-name, and in that case recordTypes will contain exactly # one recordType, so using recordTypes[0] here is always safe: memcachekey = self._memcacher.generateMemcacheKey(indexType, key) try: self._memcacher.memcacheSet("-%s" % (memcachekey,), timestamp) except DirectoryMemcacheError: log.error("Memcache: failed to store -%s" % (memcachekey,)) pass log.debug( "Directory negative cache: {index} {key}", index=indexType.value, key=key ) def purgeRecord(self, record): """ Remove a record from all indices in the cache @param record: the directory record """ if record.uid in self._cache[IndexType.uid]: del self._cache[IndexType.uid][record.uid] try: if record.guid in self._cache[IndexType.guid]: del self._cache[IndexType.guid][record.guid] except AttributeError: pass try: typeName = record.recordType.name for name in record.shortNames: key = (typeName, name) if key in self._cache[IndexType.shortName]: del self._cache[IndexType.shortName][key] except AttributeError: pass try: for emailAddress in record.emailAddresses: if emailAddress in self._cache[IndexType.emailAddress]: del self._cache[IndexType.emailAddress][emailAddress] except AttributeError: pass def purgeExpiredRecords(self): """ Scans the cache for expired records and deletes them """ if hasattr(self, "_test_time"): now = self._test_time else: now = time.time() for indexType in self._cache: for key, (cachedTime, _ignore_record) in self._cache[indexType].items(): if now - self._expireSeconds > cachedTime: del self._cache[indexType][key] def lookupRecord(self, indexType, key, name): """ Looks for a record in the specified index, under the specified key. After every config.DirectoryCaching.LookupsBetweenPurges lookups are done, purgeExpiredRecords() is called. @param index: an index type @type indexType: L{IndexType} @param key: the key to look up in the specified index @type key: any valid type that can be used as a dictionary key @return: tuple of (the cached L{DirectoryRecord}, or L{None}) and a L{bool} indicating whether a query will be required (not required if a negative cache hit) @rtype: L{tuple} """ if self._purgingEnabled: if self._lookupsUntilScan == 0: self._lookupsUntilScan = self._lookupsBetweenPurges self.purgeExpiredRecords() else: self._lookupsUntilScan -= 1 if hasattr(self, "_test_time"): now = self._test_time else: now = time.time() self._requestCount += 1 if key in self._cache[indexType]: cachedTime, record = self._cache[indexType].get(key, (0.0, None)) if now - self._expireSeconds > cachedTime: log.debug( "Directory cache miss (expired): {index} {key}", index=indexType.value, key=key ) # This record has expired self.purgeRecord(record) self._addTiming("{}-expired".format(name), 0) # Fall through when the in-memory cache expires so that we check memcache # for a valid record BEFORE we try an ldap query and recache else: log.debug( "Directory cache hit: {index} {key}", index=indexType.value, key=key ) self._hitCount += 1 self._addTiming("{}-hit".format(name), 0) return (record, False,) # Check negative cache (take cache entry timeout into account) if self.negativeCaching: try: disabledTime = self._negativeCache[indexType][key] if now - disabledTime < self._expireSeconds: log.debug( "Directory negative cache hit: {index} {key}", index=indexType.value, key=key ) self._addTiming("{}-neg-hit".format(name), 0) return (None, False,) else: del self._negativeCache[indexType][key] except KeyError: pass # Check memcache if self._memcacher is not None: # The only time the recordType arg matters is when indexType is # short-name, and in that case recordTypes will contain exactly # one recordType, so using recordTypes[0] here is always safe: memcachekey = self._memcacher.generateMemcacheKey(indexType, key) log.debug("Memcache: checking %s" % (memcachekey,)) try: record = self._memcacher.memcacheGetRecord(memcachekey) except DirectoryMemcacheError: log.error("Memcache: failed to get %s" % (memcachekey,)) record = None if record is None: log.debug("Memcache: miss %s" % (memcachekey,)) else: log.debug("Memcache: hit %s" % (memcachekey,)) self.cacheRecord(record, (IndexType.uid, IndexType.guid, IndexType.shortName,), addToMemcache=False) return (record, False,) # Check negative memcache if self.negativeCaching: try: val = self._memcacher.memcacheGet("-%s" % (memcachekey,)) except DirectoryMemcacheError: log.error("Memcache: failed to get -%s" % (memcachekey,)) val = None if val == 1: log.debug("Memcache: negative hit %s" % (memcachekey,)) self._negativeCache[indexType][key] = now return (None, False,) log.debug( "Directory cache miss: {index} {key}", index=indexType.value, key=key ) self._addTiming("{}-miss".format(name), 0) return (None, True,) # Cached methods: @inlineCallbacks def recordWithUID(self, uid, timeoutSeconds=None): # First check our cache record, doQuery = self.lookupRecord(IndexType.uid, uid, "recordWithUID") if record is None and doQuery: record = yield self._directory._wrapped_recordWithUID( uid, timeoutSeconds=timeoutSeconds ) if record is not None: # Note we do not index on email address; see below. self.cacheRecord( record, (IndexType.uid, IndexType.guid, IndexType.shortName) ) else: self.negativeCacheRecord(IndexType.uid, uid) returnValue(record) @inlineCallbacks def recordWithGUID(self, guid, timeoutSeconds=None): # First check our cache record, doQuery = self.lookupRecord(IndexType.guid, guid, "recordWithGUID") if record is None and doQuery: record = yield self._directory._wrapped_recordWithGUID( guid, timeoutSeconds=timeoutSeconds ) if record is not None: # Note we do not index on email address; see below. self.cacheRecord( record, (IndexType.uid, IndexType.guid, IndexType.shortName) ) else: self.negativeCacheRecord(IndexType.guid, guid) returnValue(record) @inlineCallbacks def recordWithShortName(self, recordType, shortName, timeoutSeconds=None): # First check our cache record, doQuery = self.lookupRecord( IndexType.shortName, (recordType.name, shortName), "recordWithShortName" ) if record is None and doQuery: record = yield self._directory._wrapped_recordWithShortName( recordType, shortName, timeoutSeconds=timeoutSeconds ) if record is not None: # Note we do not index on email address; see below. self.cacheRecord( record, (IndexType.uid, IndexType.guid, IndexType.shortName) ) else: self.negativeCacheRecord(IndexType.shortName, (recordType.name, shortName)) returnValue(record) @inlineCallbacks def recordsWithEmailAddress( self, emailAddress, limitResults=None, timeoutSeconds=None ): # First check our cache record, doQuery = self.lookupRecord( IndexType.emailAddress, emailAddress, "recordsWithEmailAddress" ) if record is None and doQuery: records = yield self._directory._wrapped_recordsWithEmailAddress( emailAddress, limitResults=limitResults, timeoutSeconds=timeoutSeconds ) if len(records) == 1: # Only cache if there was a single match (which is the most # common scenario). Caching multiple records for the exact # same key/value complicates the data structures. # Also, this is the only situation where we do index a cached # record on email address. Otherwise, say we had faulted in # on "uid" and then indexed that record on its email address, # the next lookup by email address would only get that record, # but there might be others in the directory service with that # same email address. self.cacheRecord( list(records)[0], ( IndexType.uid, IndexType.guid, IndexType.shortName, IndexType.emailAddress ) ) elif len(records) == 0: self.negativeCacheRecord(IndexType.emailAddress, emailAddress) else: records = [record] returnValue(records) # Uncached methods: @property def recordType(self): # Defer to the directory service we're caching return self._directory.recordType def recordTypes(self): # Defer to the directory service we're caching return self._directory.recordTypes() def recordsFromExpression( self, expression, recordTypes=None, records=None, limitResults=None, timeoutSeconds=None ): # Defer to the directory service we're caching return self._directory.recordsFromExpression( expression, recordTypes=recordTypes, records=records, limitResults=limitResults, timeoutSeconds=timeoutSeconds ) def recordsWithFieldValue( self, fieldName, value, limitResults=None, timeoutSeconds=None ): # Defer to the directory service we're caching return self._directory.recordsWithFieldValue( fieldName, value, limitResults=limitResults, timeoutSeconds=timeoutSeconds ) def updateRecords(self, records, create=False): # Defer to the directory service we're caching return self._directory.updateRecords(records, create=create) def removeRecords(self, uids): # Defer to the directory service we're caching return self._directory.removeRecords(uids) def recordsWithRecordType( self, recordType, limitResults=None, timeoutSeconds=None ): # Defer to the directory service we're caching return self._directory.recordsWithRecordType( recordType, limitResults=limitResults, timeoutSeconds=timeoutSeconds ) def recordsMatchingTokens( self, tokens, context=None, limitResults=None, timeoutSeconds=None ): return self._directory.recordsMatchingTokens( tokens, context=context, limitResults=limitResults, timeoutSeconds=timeoutSeconds ) def recordsMatchingFields( self, fields, operand, recordType, limitResults=None, timeoutSeconds=None ): return self._directory.recordsMatchingFields( fields, operand, recordType, limitResults=limitResults, timeoutSeconds=timeoutSeconds ) def recordsWithDirectoryBasedDelegates(self): return self._directory.recordsWithDirectoryBasedDelegates() def recordWithCalendarUserAddress(self, cua, timeoutSeconds=None): # This will get cached by the underlying recordWith... call return CalendarDirectoryServiceMixin.recordWithCalendarUserAddress( self, cua, timeoutSeconds=timeoutSeconds ) def serversDB(self): return self._directory.serversDB() @inlineCallbacks def flush(self): if self._memcacher is not None: self._memcacher.flush() self.resetCache() yield self._directory.flush() def stats(self): return self._directory.stats()
class DirectoryService(BaseDirectoryService): """ Mac OS X Server Wiki directory service. """ uidPrefix = u"wiki-" recordType = RecordType fieldName = ConstantsContainer(( BaseFieldName, FieldName, )) def __init__(self, realmName, endpointDescriptor): BaseDirectoryService.__init__(self, realmName) self.endpointDescriptor = endpointDescriptor self._recordsByName = {} # This directory service is rather limited in its skills. # We don't attempt to implement any expression handling (ie. # recordsFromNonCompoundExpression), and only support a couple of the # recordWith* convenience methods. def _recordWithName(self, name): record = self._recordsByName.get(name) if record is not None: return succeed(record) # FIXME: RPC to the wiki and check for existance of a wiki with the # given name... # # NOTE: Don't use the config module here; pass whatever info we need to # __init__(). wikiExists = True if wikiExists: record = DirectoryRecord( self, { self.fieldName.uid: u"{}{}".format(self.uidPrefix, name), self.fieldName.recordType: RecordType.macOSXServerWiki, self.fieldName.shortNames: [name], self.fieldName.fullNames: [u"Wiki: {}".format(name)], }) self._recordsByName[name] = record return succeed(record) return succeed(None) def recordWithUID(self, uid, timeoutSeconds=None): if uid.startswith(self.uidPrefix): return self._recordWithName(uid[len(self.uidPrefix):]) return succeed(None) def recordWithShortName(self, recordType, shortName, timeoutSeconds=None): if recordType is RecordType.macOSXServerWiki: return self._recordWithName(shortName) return succeed(None) def recordsFromExpression(self, expression, recordTypes=None, records=None, limitResults=None, timeoutSeconds=None): return succeed(())
def buildDirectory(store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo, serversDB=None): """ Return a directory without using a config object; suitable for tests which need to have mulitple directory instances. @param store: The store. @param dataRoot: The path to the directory containing xml files for any xml based services. @param servicesInfo: An interable of ConfigDicts mirroring the DirectoryService and ResourceService sections of stdconfig @param augmentServiceInfo: A ConfigDict mirroring the AugmentService section of stdconfig @param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig @param serversDB: A ServersDB object to assign to the directory """ aggregatedServices = [] for serviceValue in servicesInfo: if not serviceValue.Enabled: continue directoryType = serviceValue.type.lower() params = serviceValue.params if "xml" in directoryType: xmlFile = params.xmlFile xmlFile = fullServerPath(dataRoot, xmlFile) fp = FilePath(xmlFile) if not fp.exists(): fp.setContent(DEFAULT_XML_CONTENT) directory = XMLDirectoryService(fp) elif "opendirectory" in directoryType: from txdav.who.opendirectory import (DirectoryService as ODDirectoryService) # We don't want system accounts returned in lookups, so tell # the service to suppress them. directory = ODDirectoryService(suppressSystemRecords=True) elif "ldap" in directoryType: if params.credentials.dn and params.credentials.password: creds = UsernamePassword(params.credentials.dn, params.credentials.password) else: creds = None directory = LDAPDirectoryService( params.uri, params.rdnSchema.base, credentials=creds, fieldNameToAttributesMap=MappingProxyType({ BaseFieldName.uid: ("apple-generateduid", ), BaseFieldName.guid: ("apple-generateduid", ), BaseFieldName.shortNames: (LDAPAttribute.uid.value, ), BaseFieldName.fullNames: (LDAPAttribute.cn.value, ), BaseFieldName.emailAddresses: (LDAPAttribute.mail.value, ), BaseFieldName.password: (LDAPAttribute.userPassword.value, ), LDAPFieldName.memberDNs: (LDAPAttribute.uniqueMember.value, ), }), recordTypeSchemas=MappingProxyType({ RecordType.user: RecordTypeSchema( relativeDN=u"ou=People", # (objectClass=inetOrgPerson) attributes=(( LDAPAttribute.objectClass.value, LDAPObjectClass.inetOrgPerson.value, ), ), ), RecordType.group: RecordTypeSchema( relativeDN=u"ou=Groups", # (objectClass=groupOfNames) attributes=(( LDAPAttribute.objectClass.value, LDAPObjectClass.groupOfUniqueNames.value, ), ), ), })) elif "inmemory" in directoryType: from txdav.who.test.support import CalendarInMemoryDirectoryService directory = CalendarInMemoryDirectoryService() else: log.error("Invalid DirectoryType: {dt}", dt=directoryType) raise DirectoryConfigurationError # Set the appropriate record types on each service types = [] fieldNames = [] for recordTypeName in params.recordTypes: recordType = { "users": RecordType.user, "groups": RecordType.group, "locations": CalRecordType.location, "resources": CalRecordType.resource, "addresses": CalRecordType.address, }.get(recordTypeName, None) if recordType is None: log.error("Invalid Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError if recordType in types: log.error("Duplicate Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError types.append(recordType) directory.recordType = ConstantsContainer(types) directory.fieldName = ConstantsContainer( (directory.fieldName, CalFieldName)) fieldNames.append(directory.fieldName) aggregatedServices.append(directory) # # Setup the Augment Service # if augmentServiceInfo.type: for augmentFile in augmentServiceInfo.params.xmlFiles: augmentFile = fullServerPath(dataRoot, augmentFile) augmentFilePath = FilePath(augmentFile) if not augmentFilePath.exists(): augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT) augmentClass = namedClass(augmentServiceInfo.type) log.info("Configuring augment service of type: {augmentClass}", augmentClass=augmentClass) try: augmentService = augmentClass(**augmentServiceInfo.params) except IOError: log.error("Could not start augment service") raise else: augmentService = None userDirectory = None for directory in aggregatedServices: if RecordType.user in directory.recordTypes(): userDirectory = directory break else: log.error("No directory service set up for users") raise DirectoryConfigurationError # Delegate service delegateDirectory = DelegateDirectoryService(userDirectory.realmName, store) aggregatedServices.append(delegateDirectory) # Wiki service if wikiServiceInfo.Enabled: aggregatedServices.append( WikiDirectoryService(userDirectory.realmName, wikiServiceInfo.CollabHost, wikiServiceInfo.CollabPort)) # Aggregate service aggregateDirectory = AggregateDirectoryService(userDirectory.realmName, aggregatedServices) # Augment service try: fieldNames.append(CalFieldName) augmented = AugmentedDirectoryService(aggregateDirectory, store, augmentService) augmented.fieldName = ConstantsContainer(fieldNames) # The delegate directory needs a way to look up user/group records # so hand it a reference to the augmented directory. # FIXME: is there a better pattern to use here? delegateDirectory.setMasterDirectory(augmented) except Exception as e: log.error("Could not create directory service", error=e) raise if serversDB is not None: augmented.setServersDB(serversDB) return augmented
class DirectoryService(BaseDirectoryService, CalendarDirectoryServiceMixin): """ Client side of directory proxy """ # FIXME: somehow these should come from the actual directory: recordType = ConstantsContainer( (twext.who.idirectory.RecordType, txdav.who.idirectory.RecordType, txdav.who.delegates.RecordType, txdav.who.wiki.RecordType)) fieldName = ConstantsContainer( (twext.who.idirectory.FieldName, txdav.who.idirectory.FieldName, txdav.who.augment.FieldName)) def _dictToRecord(self, serializedFields): """ Turn a dictionary of fields sent from the server into a directory record """ if not serializedFields: return None # print("FIELDS", serializedFields) fields = {} for fieldName, value in serializedFields.iteritems(): try: field = self.fieldName.lookupByName(fieldName) except ValueError: # unknown field pass else: valueType = self.fieldName.valueType(field) if valueType in (unicode, bool): fields[field] = value elif valueType is uuid.UUID: fields[field] = uuid.UUID(value) elif issubclass(valueType, Names): if value is not None: fields[field] = field.valueType.lookupByName(value) else: fields[field] = None elif issubclass(valueType, NamedConstant): if fieldName == "recordType": # Is there a better way? fields[field] = self.recordType.lookupByName(value) # print("AFTER:", fields) return DirectoryRecord(self, fields) def _processSingleRecord(self, result): """ Takes a dictionary with a "fields" key whose value is a pickled dictionary of a record's fields, and returns a record. """ serializedFields = pickle.loads(result['fields']) return self._dictToRecord(serializedFields) def _processMultipleRecords(self, result): """ Takes a dictionary with a "items" key whose value is an iterable of pickled dictionaries (of records' fields), and returns a list of records. """ serializedFieldsList = [] for fields in result["items"]: fields = pickle.loads(fields) serializedFieldsList.append(fields) results = [] for serializedFields in serializedFieldsList: record = self._dictToRecord(serializedFields) if record is not None: results.append(record) return results @inlineCallbacks def _getConnection(self): from twistedcaldav.config import config path = config.DirectoryProxy.SocketPath if getattr(self, "_connection", None) is None: log.debug("Creating connection") connection = (yield ClientCreator(reactor, amp.AMP).connectUNIX(path)) self._connection = connection returnValue(self._connection) @inlineCallbacks def _sendCommand(self, command, **kwds): """ Execute a remote AMP command, first making the connection to the peer. Any kwds are passed on to the AMP command. @param command: the AMP command to call @type command: L{twisted.protocols.amp.Command} """ ampProto = (yield self._getConnection()) try: results = (yield ampProto.callRemote(command, **kwds)) except Exception, e: log.error("Failed AMP command", error=e) # FIXME: is there a way to hook into ConnectionLost? self._connection = None raise returnValue(results)
class CachingDirectoryService(BaseDirectoryService, CalendarDirectoryServiceMixin): """ Caching directory service. This is a directory service that wraps an L{IDirectoryService} and caches directory records. """ fieldName = ConstantsContainer(( BaseFieldName, FieldName, )) def __init__(self, directory, expireSeconds=30): BaseDirectoryService.__init__(self, directory.realmName) self._directory = directory # Patch the wrapped directory service's recordWithXXX to instead # use this cache directory._wrapped_recordWithUID = directory.recordWithUID directory.recordWithUID = self.recordWithUID directory._wrapped_recordWithGUID = directory.recordWithGUID directory.recordWithGUID = self.recordWithGUID directory._wrapped_recordWithShortName = directory.recordWithShortName directory.recordWithShortName = self.recordWithShortName directory._wrapped_recordsWithEmailAddress = directory.recordsWithEmailAddress directory.recordsWithEmailAddress = self.recordsWithEmailAddress self._expireSeconds = expireSeconds self.resetCache() def setTimingMethod(self, f): """ Replace the default no-op timing method """ self._addTiming = f def _addTiming(self, key, duration): """ Timing won't get recorded by default -- you must call setTimingMethod with a callable that takes a key such as a method name, and a duration. """ pass def resetCache(self): """ Clear the cache """ self._cache = { IndexType.uid: {}, IndexType.guid: {}, IndexType.shortName: {}, # key is (recordType.name, shortName) IndexType.emailAddress: {}, } self._hitCount = 0 self._requestCount = 0 self._lookupsUntilScan = SCAN_AFTER_LOOKUP_COUNT def setTestTime(self, timestamp): """ Only used for unit tests to override the notion of "now" @param timestamp: seconds @type timestamp: C{float} """ self._test_time = timestamp def cacheRecord(self, record, indexTypes): """ Store a record in the cache, within the specified indexes @param record: the directory record @param indexTypes: an iterable of L{IndexType} """ if hasattr(self, "_test_time"): timestamp = self._test_time else: timestamp = time.time() if IndexType.uid in indexTypes: self._cache[IndexType.uid][record.uid] = (timestamp, record) if IndexType.guid in indexTypes: try: self._cache[IndexType.guid][record.guid] = (timestamp, record) except AttributeError: pass if IndexType.shortName in indexTypes: try: typeName = record.recordType.name for name in record.shortNames: self._cache[IndexType.shortName][(typeName, name)] = (timestamp, record) except AttributeError: pass if IndexType.emailAddress in indexTypes: try: for emailAddress in record.emailAddresses: self._cache[IndexType.emailAddress][emailAddress] = ( timestamp, record) except AttributeError: pass def purgeRecord(self, record): """ Remove a record from all indices in the cache @param record: the directory record """ if record.uid in self._cache[IndexType.uid]: del self._cache[IndexType.uid][record.uid] try: if record.guid in self._cache[IndexType.guid]: del self._cache[IndexType.guid][record.guid] except AttributeError: pass try: typeName = record.recordType.name for name in record.shortNames: key = (typeName, name) if key in self._cache[IndexType.shortName]: del self._cache[IndexType.shortName][key] except AttributeError: pass try: for emailAddress in record.emailAddresses: if emailAddress in self._cache[IndexType.emailAddress]: del self._cache[IndexType.emailAddress][emailAddress] except AttributeError: pass def purgeExpiredRecords(self): """ Scans the cache for expired records and deletes them """ if hasattr(self, "_test_time"): now = self._test_time else: now = time.time() for indexType in self._cache: for key, (cachedTime, _ignore_record) in self._cache[indexType].items(): if now - self._expireSeconds > cachedTime: del self._cache[indexType][key] def lookupRecord(self, indexType, key, name): """ Looks for a record in the specified index, under the specified key. After every SCAN_AFTER_LOOKUP_COUNT lookups are done, purgeExpiredRecords() is called. @param index: an index type @type indexType: L{IndexType} @param key: the key to look up in the specified index @type key: any valid type that can be used as a dictionary key @return: the cached directory record, or None @rtype: L{DirectoryRecord} """ if self._lookupsUntilScan == 0: self._lookupsUntilScan = SCAN_AFTER_LOOKUP_COUNT self.purgeExpiredRecords() else: self._lookupsUntilScan -= 1 self._requestCount += 1 if key in self._cache[indexType]: if hasattr(self, "_test_time"): now = self._test_time else: now = time.time() cachedTime, record = self._cache[indexType].get(key, (0.0, None)) if now - self._expireSeconds > cachedTime: log.debug("Directory cache miss (expired): {index} {key}", index=indexType.value, key=key) # This record has expired self.purgeRecord(record) self._addTiming("{}-expired".format(name), 0) return None log.debug("Directory cache hit: {index} {key}", index=indexType.value, key=key) self._hitCount += 1 self._addTiming("{}-hit".format(name), 0) return record else: log.debug("Directory cache miss: {index} {key}", index=indexType.value, key=key) self._addTiming("{}-miss".format(name), 0) return None # Cached methods: @inlineCallbacks def recordWithUID(self, uid, timeoutSeconds=None): # First check our cache record = self.lookupRecord(IndexType.uid, uid, "recordWithUID") if record is None: record = yield self._directory._wrapped_recordWithUID( uid, timeoutSeconds=timeoutSeconds) if record is not None: # Note we do not index on email address; see below. self.cacheRecord( record, (IndexType.uid, IndexType.guid, IndexType.shortName)) returnValue(record) @inlineCallbacks def recordWithGUID(self, guid, timeoutSeconds=None): # First check our cache record = self.lookupRecord(IndexType.guid, guid, "recordWithGUID") if record is None: record = yield self._directory._wrapped_recordWithGUID( guid, timeoutSeconds=timeoutSeconds) if record is not None: # Note we do not index on email address; see below. self.cacheRecord( record, (IndexType.uid, IndexType.guid, IndexType.shortName)) returnValue(record) @inlineCallbacks def recordWithShortName(self, recordType, shortName, timeoutSeconds=None): # First check our cache record = self.lookupRecord(IndexType.shortName, (recordType.name, shortName), "recordWithShortName") if record is None: record = yield self._directory._wrapped_recordWithShortName( recordType, shortName, timeoutSeconds=timeoutSeconds) if record is not None: # Note we do not index on email address; see below. self.cacheRecord( record, (IndexType.uid, IndexType.guid, IndexType.shortName)) returnValue(record) @inlineCallbacks def recordsWithEmailAddress(self, emailAddress, limitResults=None, timeoutSeconds=None): # First check our cache record = self.lookupRecord(IndexType.emailAddress, emailAddress, "recordsWithEmailAddress") if record is None: records = yield self._directory._wrapped_recordsWithEmailAddress( emailAddress, limitResults=limitResults, timeoutSeconds=timeoutSeconds) if len(records) == 1: # Only cache if there was a single match (which is the most # common scenario). Caching multiple records for the exact # same key/value complicates the data structures. # Also, this is the only situation where we do index a cached # record on email address. Otherwise, say we had faulted in # on "uid" and then indexed that record on its email address, # the next lookup by email address would only get that record, # but there might be others in the directory service with that # same email address. self.cacheRecord( list(records)[0], (IndexType.uid, IndexType.guid, IndexType.shortName, IndexType.emailAddress)) else: records = [record] returnValue(records) # Uncached methods: @property def recordType(self): # Defer to the directory service we're caching return self._directory.recordType def recordTypes(self): # Defer to the directory service we're caching return self._directory.recordTypes() def recordsFromExpression(self, expression, recordTypes=None, records=None, limitResults=None, timeoutSeconds=None): # Defer to the directory service we're caching return self._directory.recordsFromExpression( expression, recordTypes=recordTypes, records=records, limitResults=limitResults, timeoutSeconds=timeoutSeconds) def recordsWithFieldValue(self, fieldName, value, limitResults=None, timeoutSeconds=None): # Defer to the directory service we're caching return self._directory.recordsWithFieldValue( fieldName, value, limitResults=limitResults, timeoutSeconds=timeoutSeconds) def updateRecords(self, records, create=False): # Defer to the directory service we're caching return self._directory.updateRecords(records, create=create) def removeRecords(self, uids): # Defer to the directory service we're caching return self._directory.removeRecords(uids) def recordsWithRecordType(self, recordType, limitResults=None, timeoutSeconds=None): # Defer to the directory service we're caching return self._directory.recordsWithRecordType( recordType, limitResults=limitResults, timeoutSeconds=timeoutSeconds) def recordsMatchingTokens(self, tokens, context=None, limitResults=None, timeoutSeconds=None): return self._directory.recordsMatchingTokens( tokens, context=context, limitResults=limitResults, timeoutSeconds=timeoutSeconds) def recordsMatchingFields(self, fields, operand, recordType, limitResults=None, timeoutSeconds=None): return self._directory.recordsMatchingFields( fields, operand, recordType, limitResults=limitResults, timeoutSeconds=timeoutSeconds) def recordsWithDirectoryBasedDelegates(self): return self._directory.recordsWithDirectoryBasedDelegates() def recordWithCalendarUserAddress(self, cua, timeoutSeconds=None): # This will get cached by the underlying recordWith... call return CalendarDirectoryServiceMixin.recordWithCalendarUserAddress( self, cua, timeoutSeconds=timeoutSeconds) def serversDB(self): return self._directory.serversDB() @inlineCallbacks def flush(self): self.resetCache() yield self._directory.flush() def stats(self): return self._directory.stats()
class AugmentedDirectoryService(BaseDirectoryService, CalendarDirectoryServiceMixin): """ Augmented directory service. This is a directory service that wraps an L{IDirectoryService} and augments directory records with additional or modified fields. """ fieldName = ConstantsContainer(( BaseFieldName, FieldName, )) _timings = {} def __init__(self, directory, store, augmentDB): BaseDirectoryService.__init__(self, directory.realmName) self._directory = directory self._store = store self._augmentDB = augmentDB # An LDAP DS has extra info to expose via the dashboard # This is assigned in buildDirectory() self._ldapDS = None @classmethod def _addTiming(cls, key, duration): if key not in cls._timings: cls._timings[key] = (0, 0.0) count, timeSpent = cls._timings[key] count += 1 timeSpent += duration cls._timings[key] = (count, timeSpent) def flush(self): return self._directory.flush() def stats(self): results = {} results.update(self._timings) # An LDAP DS has extra info to expose via the dashboard if self._ldapDS is not None: results.update(self._ldapDS.poolStats) return succeed(results) @property def recordType(self): # Defer to the directory service we're augmenting return self._directory.recordType def recordTypes(self): # Defer to the directory service we're augmenting return self._directory.recordTypes() @inlineCallbacks def recordsFromExpression(self, expression, recordTypes=None, limitResults=None, timeoutSeconds=None): records = yield self._directory.recordsFromExpression( expression, recordTypes=recordTypes, limitResults=limitResults, timeoutSeconds=timeoutSeconds) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @inlineCallbacks def recordsWithFieldValue(self, fieldName, value, limitResults=None, timeoutSeconds=None): records = yield self._directory.recordsWithFieldValue( fieldName, value, limitResults=limitResults, timeoutSeconds=timeoutSeconds) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @timed @inlineCallbacks def recordWithUID(self, uid, timeoutSeconds=None): # MOVE2WHO, REMOVE THIS: if not isinstance(uid, unicode): # log.warn("Need to change uid to unicode") uid = uid.decode("utf-8") record = yield self._directory.recordWithUID( uid, timeoutSeconds=timeoutSeconds) record = yield self._augment(record) returnValue(record) @timed @inlineCallbacks def recordWithGUID(self, guid, timeoutSeconds=None): record = yield self._directory.recordWithGUID( guid, timeoutSeconds=timeoutSeconds) record = yield self._augment(record) returnValue(record) @timed @inlineCallbacks def recordsWithRecordType(self, recordType, limitResults=None, timeoutSeconds=None): records = yield self._directory.recordsWithRecordType( recordType, limitResults=limitResults, timeoutSeconds=timeoutSeconds) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @timed @inlineCallbacks def recordWithShortName(self, recordType, shortName, timeoutSeconds=None): # MOVE2WHO, REMOVE THIS: if not isinstance(shortName, unicode): # log.warn("Need to change shortName to unicode") shortName = shortName.decode("utf-8") record = yield self._directory.recordWithShortName( recordType, shortName, timeoutSeconds=timeoutSeconds) record = yield self._augment(record) returnValue(record) @timed @inlineCallbacks def recordsWithEmailAddress(self, emailAddress, limitResults=None, timeoutSeconds=None): # MOVE2WHO, REMOVE THIS: if not isinstance(emailAddress, unicode): # log.warn("Need to change emailAddress to unicode") emailAddress = emailAddress.decode("utf-8") records = yield self._directory.recordsWithEmailAddress( emailAddress, limitResults=limitResults, timeoutSeconds=timeoutSeconds) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @timed def recordWithCalendarUserAddress(self, *args, **kwds): return CalendarDirectoryServiceMixin.recordWithCalendarUserAddress( self, *args, **kwds) @timed def recordsMatchingTokens(self, *args, **kwds): return CalendarDirectoryServiceMixin.recordsMatchingTokens( self, *args, **kwds) @timed def recordsMatchingFields(self, *args, **kwds): return CalendarDirectoryServiceMixin.recordsMatchingFields( self, *args, **kwds) @timed @inlineCallbacks def updateRecords(self, records, create=False): """ Pull out the augmented fields from each record, apply those to the augments database, then update the base records. """ baseRecords = [] augmentRecords = [] for record in records: # Split out the base fields from the augment fields baseFields, augmentFields = self._splitFields(record) # Ignore groups for now if augmentFields and record.recordType != RecordType.group: # Create an AugmentRecord autoScheduleMode = { AutoScheduleMode.none: "none", AutoScheduleMode.accept: "accept-always", AutoScheduleMode.decline: "decline-always", AutoScheduleMode.acceptIfFree: "accept-if-free", AutoScheduleMode.declineIfBusy: "decline-if-busy", AutoScheduleMode.acceptIfFreeDeclineIfBusy: "automatic", }.get(augmentFields.get(FieldName.autoScheduleMode, None), None) kwargs = { "uid": record.uid, "autoScheduleMode": autoScheduleMode, } if FieldName.hasCalendars in augmentFields: kwargs["enabledForCalendaring"] = augmentFields[ FieldName.hasCalendars] if FieldName.hasContacts in augmentFields: kwargs["enabledForAddressBooks"] = augmentFields[ FieldName.hasContacts] if FieldName.loginAllowed in augmentFields: kwargs["enabledForLogin"] = augmentFields[ FieldName.loginAllowed] if FieldName.autoAcceptGroup in augmentFields: kwargs["autoAcceptGroup"] = augmentFields[ FieldName.autoAcceptGroup] if FieldName.serviceNodeUID in augmentFields: kwargs["serverID"] = augmentFields[ FieldName.serviceNodeUID] augmentRecord = AugmentRecord(**kwargs) augmentRecords.append(augmentRecord) # Create new base records: baseRecords.append( DirectoryRecord( self._directory, record._baseRecord.fields if hasattr( record, "_baseRecord") else baseFields)) # Apply the augment records if augmentRecords: yield self._augmentDB.addAugmentRecords(augmentRecords) # Apply the base records if baseRecords: try: yield self._directory.updateRecords(baseRecords, create=create) except NotAllowedError: pass def _splitFields(self, record): """ Returns a tuple of two dictionaries; the first contains all the non augment fields, and the second contains all the augment fields. """ if record is None: return None augmentFields = {} baseFields = record.fields.copy() for field in (FieldName.loginAllowed, FieldName.hasCalendars, FieldName.hasContacts, FieldName.autoScheduleMode, FieldName.autoAcceptGroup, FieldName.serviceNodeUID): if field in baseFields: augmentFields[field] = baseFields[field] del baseFields[field] return (baseFields, augmentFields) @inlineCallbacks def removeRecords(self, uids): yield self._augmentDB.removeAugmentRecords(uids) yield self._directory.removeRecords(uids) def _assignToField(self, fields, name, value): """ Assign a value to a field only if not already present in fields. """ field = self.fieldName.lookupByName(name) if field not in fields: fields[field] = value @inlineCallbacks def _augment(self, record): if record is None: returnValue(None) augmentRecord = yield self._augmentDB.getAugmentRecord( record.uid, self.recordTypeToOldName(record.recordType)) if augmentRecord is None: # Augments does not know about this record type, so return # the original record returnValue(record) fields = record.fields.copy() if augmentRecord: if record.recordType == RecordType.group: self._assignToField(fields, "hasCalendars", False) self._assignToField(fields, "hasContacts", False) else: self._assignToField(fields, "hasCalendars", augmentRecord.enabledForCalendaring) self._assignToField(fields, "hasContacts", augmentRecord.enabledForAddressBooks) # In the case of XML augments, a missing auto-schedule-mode # element has the same meaning an element with a value of "default" # in which case augmentRecord.autoScheduleMode = "default". On # the record we're augmenting, "default" mode means autoScheduleMode # gets set to None (distinct from AutoScheduleMode.none!), # which gets swapped for config.Scheduling.Options.AutoSchedule.DefaultMode # in checkAttendeeAutoReply(). # ...Except for locations/resources which will default to automatic autoScheduleMode = { "none": AutoScheduleMode.none, "accept-always": AutoScheduleMode.accept, "decline-always": AutoScheduleMode.decline, "accept-if-free": AutoScheduleMode.acceptIfFree, "decline-if-busy": AutoScheduleMode.declineIfBusy, "automatic": AutoScheduleMode.acceptIfFreeDeclineIfBusy, }.get(augmentRecord.autoScheduleMode, None) # Resources/Locations default to automatic if record.recordType in (CalRecordType.location, CalRecordType.resource): if autoScheduleMode is None: autoScheduleMode = AutoScheduleMode.acceptIfFreeDeclineIfBusy self._assignToField(fields, "autoScheduleMode", autoScheduleMode) if augmentRecord.autoAcceptGroup is not None: self._assignToField( fields, "autoAcceptGroup", augmentRecord.autoAcceptGroup.decode("utf-8")) self._assignToField(fields, "loginAllowed", augmentRecord.enabledForLogin) self._assignToField(fields, "serviceNodeUID", augmentRecord.serverID.decode("utf-8")) else: self._assignToField(fields, "hasCalendars", False) self._assignToField(fields, "hasContacts", False) self._assignToField(fields, "loginAllowed", False) # print("Augmented fields", fields) # Clone to a new record with the augmented fields augmentedRecord = AugmentedDirectoryRecord(self, record, fields) returnValue(augmentedRecord) @inlineCallbacks def setAutoScheduleMode(self, record, autoScheduleMode): augmentRecord = yield self._augmentDB.getAugmentRecord( record.uid, self.recordTypeToOldName(record.recordType)) if augmentRecord is not None: autoScheduleMode = { AutoScheduleMode.none: "none", AutoScheduleMode.accept: "accept-always", AutoScheduleMode.decline: "decline-always", AutoScheduleMode.acceptIfFree: "accept-if-free", AutoScheduleMode.declineIfBusy: "decline-if-busy", AutoScheduleMode.acceptIfFreeDeclineIfBusy: "automatic", }.get(autoScheduleMode) augmentRecord.autoScheduleMode = autoScheduleMode yield self._augmentDB.addAugmentRecords([augmentRecord])
class EnhancedDirectoryService(OpenDirectoryService): recordType = ConstantsContainer(( BaseRecordType.user, BaseRecordType.group, _CSRecordType.location, _CSRecordType.resource ))
def buildDirectory( store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo, serversDB=None, cachingSeconds=0, filterStartsWith=False, lookupsBetweenPurges=0, negativeCaching=True, ): """ Return a directory without using a config object; suitable for tests which need to have mulitple directory instances. @param store: The store. @param dataRoot: The path to the directory containing xml files for any xml based services. @param servicesInfo: An interable of ConfigDicts mirroring the DirectoryService and ResourceService sections of stdconfig @param augmentServiceInfo: A ConfigDict mirroring the AugmentService section of stdconfig @param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig @param serversDB: A ServersDB object to assign to the directory """ aggregatedServices = [] cachingServices = [] ldapService = None # LDAP DS has extra stats (see augment.py) for serviceValue in servicesInfo: if not serviceValue.Enabled: continue directoryType = serviceValue.type.lower() params = serviceValue.params if "xml" in directoryType: xmlFile = params.xmlFile xmlFile = fullServerPath(dataRoot, xmlFile) fp = FilePath(xmlFile) if not fp.exists(): fp.setContent(DEFAULT_XML_CONTENT) directory = XMLDirectoryService(fp) elif "opendirectory" in directoryType: from txdav.who.opendirectory import (DirectoryService as ODDirectoryService) # We don't want system accounts returned in lookups, so tell # the service to suppress them. node = params.node directory = ODDirectoryService(nodeName=node, suppressSystemRecords=True) elif "ldap" in directoryType: from twext.who.ldap import (DirectoryService as LDAPDirectoryService, FieldName as LDAPFieldName, RecordTypeSchema) if params.credentials.dn and params.credentials.password: creds = UsernamePassword(params.credentials.dn, params.credentials.password) else: creds = None mapping = params.mapping extraFilters = params.extraFilters directory = LDAPDirectoryService( params.uri, params.rdnSchema.base, useTLS=params.useTLS, credentials=creds, fieldNameToAttributesMap=MappingProxyType({ BaseFieldName.uid: mapping.uid, BaseFieldName.guid: mapping.guid, BaseFieldName.shortNames: mapping.shortNames, BaseFieldName.fullNames: mapping.fullNames, BaseFieldName.emailAddresses: mapping.emailAddresses, LDAPFieldName.memberDNs: mapping.memberDNs, CalFieldName.readOnlyProxy: mapping.readOnlyProxy, CalFieldName.readWriteProxy: mapping.readWriteProxy, CalFieldName.loginAllowed: mapping.loginAllowed, CalFieldName.hasCalendars: mapping.hasCalendars, CalFieldName.autoScheduleMode: mapping.autoScheduleMode, CalFieldName.autoAcceptGroup: mapping.autoAcceptGroup, CalFieldName.serviceNodeUID: mapping.serviceNodeUID, CalFieldName.associatedAddress: mapping.associatedAddress, CalFieldName.geographicLocation: mapping.geographicLocation, CalFieldName.streetAddress: mapping.streetAddress, }), recordTypeSchemas=MappingProxyType({ RecordType.user: RecordTypeSchema( relativeDN=params.rdnSchema.users, attributes=(), ), RecordType.group: RecordTypeSchema( relativeDN=params.rdnSchema.groups, attributes=(), ), CalRecordType.location: RecordTypeSchema( relativeDN=params.rdnSchema.locations, attributes=(), ), CalRecordType.resource: RecordTypeSchema( relativeDN=params.rdnSchema.resources, attributes=(), ), CalRecordType.address: RecordTypeSchema( relativeDN=params.rdnSchema.addresses, attributes=(), ), }), extraFilters={ RecordType.user: extraFilters.get("users", ""), RecordType.group: extraFilters.get("groups", ""), CalRecordType.location: extraFilters.get("locations", ""), CalRecordType.resource: extraFilters.get("resources", ""), CalRecordType.address: extraFilters.get("addresses", ""), }, threadPoolMax=params.get("threadPoolMax", 10), authConnectionMax=params.get("authConnectionMax", 5), queryConnectionMax=params.get("queryConnectionMax", 5), tries=params.get("tries", 3), warningThresholdSeconds=params.get("warningThresholdSeconds", 5), ) ldapService = directory elif "inmemory" in directoryType: from txdav.who.test.support import CalendarInMemoryDirectoryService directory = CalendarInMemoryDirectoryService() else: log.error("Invalid DirectoryType: {dt}", dt=directoryType) raise DirectoryConfigurationError # Set the appropriate record types on each service types = [] fieldNames = [] for recordTypeName in params.recordTypes: recordType = { "users": RecordType.user, "groups": RecordType.group, "locations": CalRecordType.location, "resources": CalRecordType.resource, "addresses": CalRecordType.address, }.get(recordTypeName, None) if recordType is None: log.error("Invalid Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError if recordType in types: log.error("Duplicate Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError types.append(recordType) directory.recordType = ConstantsContainer(types) directory.fieldName = ConstantsContainer( (directory.fieldName, CalFieldName)) fieldNames.append(directory.fieldName) if cachingSeconds: directory = CachingDirectoryService( directory, expireSeconds=cachingSeconds, lookupsBetweenPurges=lookupsBetweenPurges, negativeCaching=negativeCaching, ) cachingServices.append(directory) aggregatedServices.append(directory) # # Setup the Augment Service # serviceClass = { "xml": "twistedcaldav.directory.augment.AugmentXMLDB", } for augmentFile in augmentServiceInfo.params.xmlFiles: augmentFile = fullServerPath(dataRoot, augmentFile) augmentFilePath = FilePath(augmentFile) if not augmentFilePath.exists(): augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT) augmentClass = namedClass(serviceClass[augmentServiceInfo.type]) log.info("Configuring augment service of type: {augmentClass}", augmentClass=augmentClass) try: augmentService = augmentClass(**augmentServiceInfo.params) except IOError: log.error("Could not start augment service") raise userDirectory = None for directory in aggregatedServices: if RecordType.user in directory.recordTypes(): userDirectory = directory break else: log.error("No directory service set up for users") raise DirectoryConfigurationError # Delegate service delegateDirectory = DelegateDirectoryService(userDirectory.realmName, store) # (put at front of list so we don't try to ask the actual DS services # about the delegate-related principals, for performance) aggregatedServices.insert(0, delegateDirectory) # Wiki service if wikiServiceInfo.Enabled: aggregatedServices.append( WikiDirectoryService( userDirectory.realmName, wikiServiceInfo.EndpointDescriptor, )) # Aggregate service aggregateDirectory = AggregateDirectoryService(userDirectory.realmName, aggregatedServices) # Augment service try: fieldNames.append(CalFieldName) augmented = AugmentedDirectoryService(aggregateDirectory, store, augmentService) augmented.fieldName = ConstantsContainer(fieldNames) # The delegate directory needs a way to look up user/group records # so hand it a reference to the augmented directory. # FIXME: is there a better pattern to use here? delegateDirectory.setMasterDirectory(augmented) # Tell each caching service what method to use when reporting # times and cache stats for cachingService in cachingServices: cachingService.setTimingMethod(augmented._addTiming) # LDAP has additional stats to report augmented._ldapDS = ldapService except Exception as e: log.error("Could not create directory service", error=e) raise if serversDB is not None: augmented.setServersDB(serversDB) if filterStartsWith: augmented.setFilter(startswithFilter) return augmented
class AugmentedDirectoryService(BaseDirectoryService, CalendarDirectoryServiceMixin): """ Augmented directory service. This is a directory service that wraps an L{IDirectoryService} and augments directory records with additional or modified fields. """ fieldName = ConstantsContainer(( BaseFieldName, FieldName, )) _timings = {} def __init__(self, directory, store, augmentDB): BaseDirectoryService.__init__(self, directory.realmName) self._directory = directory self._store = store self._augmentDB = augmentDB def stats(self): return self._timings @property def recordType(self): # Defer to the directory service we're augmenting return self._directory.recordType def recordTypes(self): # Defer to the directory service we're augmenting return self._directory.recordTypes() @inlineCallbacks def recordsFromExpression(self, expression, recordTypes=None): records = yield self._directory.recordsFromExpression( expression, recordTypes=recordTypes) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @inlineCallbacks def recordsWithFieldValue(self, fieldName, value): records = yield self._directory.recordsWithFieldValue(fieldName, value) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @timed @inlineCallbacks def recordWithUID(self, uid): # MOVE2WHO, REMOVE THIS: if not isinstance(uid, unicode): # log.warn("Need to change uid to unicode") uid = uid.decode("utf-8") record = yield self._directory.recordWithUID(uid) record = yield self._augment(record) returnValue(record) @timed @inlineCallbacks def recordWithGUID(self, guid): record = yield self._directory.recordWithGUID(guid) record = yield self._augment(record) returnValue(record) @timed @inlineCallbacks def recordsWithRecordType(self, recordType): records = yield self._directory.recordsWithRecordType(recordType) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @timed @inlineCallbacks def recordWithShortName(self, recordType, shortName): # MOVE2WHO, REMOVE THIS: if not isinstance(shortName, unicode): # log.warn("Need to change shortName to unicode") shortName = shortName.decode("utf-8") record = yield self._directory.recordWithShortName( recordType, shortName) record = yield self._augment(record) returnValue(record) @timed @inlineCallbacks def recordsWithEmailAddress(self, emailAddress): # MOVE2WHO, REMOVE THIS: if not isinstance(emailAddress, unicode): # log.warn("Need to change emailAddress to unicode") emailAddress = emailAddress.decode("utf-8") records = yield self._directory.recordsWithEmailAddress(emailAddress) augmented = [] for record in records: record = yield self._augment(record) augmented.append(record) returnValue(augmented) @timed def recordWithCalendarUserAddress(self, *args, **kwds): return CalendarDirectoryServiceMixin.recordWithCalendarUserAddress( self, *args, **kwds) @timed def recordsMatchingTokens(self, *args, **kwds): return CalendarDirectoryServiceMixin.recordsMatchingTokens( self, *args, **kwds) @timed def recordsMatchingFields(self, *args, **kwds): return CalendarDirectoryServiceMixin.recordsMatchingFields( self, *args, **kwds) @timed @inlineCallbacks def updateRecords(self, records, create=False): """ Pull out the augmented fields from each record, apply those to the augments database, then update the base records. """ baseRecords = [] augmentRecords = [] for record in records: # Split out the base fields from the augment fields baseFields, augmentFields = self._splitFields(record) if augmentFields: # Create an AugmentRecord autoScheduleMode = { AutoScheduleMode.none: "none", AutoScheduleMode.accept: "accept-always", AutoScheduleMode.decline: "decline-always", AutoScheduleMode.acceptIfFree: "accept-if-free", AutoScheduleMode.declineIfBusy: "decline-if-busy", AutoScheduleMode.acceptIfFreeDeclineIfBusy: "automatic", }.get(augmentFields.get(FieldName.autoScheduleMode, None), None) kwargs = { "uid": record.uid, "autoScheduleMode": autoScheduleMode, } if FieldName.hasCalendars in augmentFields: kwargs["enabledForCalendaring"] = augmentFields[ FieldName.hasCalendars] if FieldName.hasContacts in augmentFields: kwargs["enabledForAddressBooks"] = augmentFields[ FieldName.hasContacts] if FieldName.loginAllowed in augmentFields: kwargs["enabledForLogin"] = augmentFields[ FieldName.loginAllowed] if FieldName.autoAcceptGroup in augmentFields: kwargs["autoAcceptGroup"] = augmentFields[ FieldName.autoAcceptGroup] if FieldName.serviceNodeUID in augmentFields: kwargs["serverID"] = augmentFields[ FieldName.serviceNodeUID] augmentRecord = AugmentRecord(**kwargs) augmentRecords.append(augmentRecord) # Create new base records: baseRecords.append(DirectoryRecord(self._directory, baseFields)) # Apply the augment records if augmentRecords: yield self._augmentDB.addAugmentRecords(augmentRecords) # Apply the base records if baseRecords: yield self._directory.updateRecords(baseRecords, create=create) def _splitFields(self, record): """ Returns a tuple of two dictionaries; the first contains all the non augment fields, and the second contains all the augment fields. """ if record is None: return None augmentFields = {} baseFields = record.fields.copy() for field in (FieldName.loginAllowed, FieldName.hasCalendars, FieldName.hasContacts, FieldName.autoScheduleMode, FieldName.autoAcceptGroup, FieldName.serviceNodeUID): if field in baseFields: augmentFields[field] = baseFields[field] del baseFields[field] return (baseFields, augmentFields) def removeRecords(self, uids): self._augmentDB.removeAugmentRecords(uids) return self._directory.removeRecords(uids) def _assignToField(self, fields, name, value): field = self.fieldName.lookupByName(name) fields[field] = value @inlineCallbacks def _augment(self, record): if record is None: returnValue(None) augmentRecord = yield self._augmentDB.getAugmentRecord( record.uid, self.recordTypeToOldName(record.recordType)) if augmentRecord is None: # Augments does not know about this record type, so return # the original record returnValue(record) fields = record.fields.copy() # print("Got augment record", augmentRecord) if augmentRecord: self._assignToField(fields, "hasCalendars", augmentRecord.enabledForCalendaring) self._assignToField(fields, "hasContacts", augmentRecord.enabledForAddressBooks) autoScheduleMode = { "none": AutoScheduleMode.none, "accept-always": AutoScheduleMode.accept, "decline-always": AutoScheduleMode.decline, "accept-if-free": AutoScheduleMode.acceptIfFree, "decline-if-busy": AutoScheduleMode.declineIfBusy, "automatic": AutoScheduleMode.acceptIfFreeDeclineIfBusy, }.get(augmentRecord.autoScheduleMode, None) # Resources/Locations default to automatic if record.recordType in (CalRecordType.location, CalRecordType.resource): if autoScheduleMode is None: autoScheduleMode = AutoScheduleMode.acceptIfFreeDeclineIfBusy self._assignToField(fields, "autoScheduleMode", autoScheduleMode) if augmentRecord.autoAcceptGroup is not None: self._assignToField( fields, "autoAcceptGroup", augmentRecord.autoAcceptGroup.decode("utf-8")) self._assignToField(fields, "loginAllowed", augmentRecord.enabledForLogin) self._assignToField(fields, "serviceNodeUID", augmentRecord.serverID.decode("utf-8")) if ((fields.get(self.fieldName.lookupByName("hasCalendars"), False) or fields.get(self.fieldName.lookupByName("hasContacts"), False)) and record.recordType == RecordType.group): self._assignToField(fields, "hasCalendars", False) self._assignToField(fields, "hasContacts", False) # For augment records cloned from the Default augment record, # don't emit this message: if not augmentRecord.clonedFromDefault: log.error( "Group {record} cannot be enabled for " "calendaring or address books", record=record) else: self._assignToField(fields, "hasCalendars", False) self._assignToField(fields, "hasContacts", False) self._assignToField(fields, "loginAllowed", False) # print("Augmented fields", fields) # Clone to a new record with the augmented fields returnValue(AugmentedDirectoryRecord(self, record, fields))