def setUp(self): self.numUsers = 1000 # The "local" directory service self.directory = DirectoryService(None) # The "remote" directory service remoteDirectory = CalendarInMemoryDirectoryService(None) # Add users records = [] fieldName = remoteDirectory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( remoteDirectory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.recordType: RecordType.user, })) # Add a big group records.append( TestRecord( remoteDirectory, { fieldName.uid: u"bigGroup", fieldName.recordType: RecordType.group, })) yield remoteDirectory.updateRecords(records, create=True) group = yield remoteDirectory.recordWithUID(u"bigGroup") members = yield remoteDirectory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) # Connect the two services directly via an IOPump client = AMP() self.server = DirectoryProxyAMPProtocol(remoteDirectory) pump = returnConnected(self.server, client) # Replace the normal _getConnection method with one that bypasses any # actual networking self.patch(self.directory, "_getConnection", lambda: succeed(client)) # Wrap the normal _call method with one that flushes the IOPump # afterwards origCall = self.directory._call def newCall(*args, **kwds): d = origCall(*args, **kwds) pump.flush() return d self.patch(self.directory, "_call", newCall)
def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.recordType: RecordType.user, } ) ) # Add two groups for uid in (u"testgroup", u"emptygroup",): records.append( TestRecord( self.directory, { fieldName.uid: uid, fieldName.recordType: RecordType.group, } ) ) yield self.directory.updateRecords(records, create=True) # add members to test group group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) def doWork(self): self.transaction._groupCacher = groupCacher return unpatchedDoWork(self) groupCacher = self.groupCacher unpatchedDoWork = GroupRefreshWork.doWork self.patch(GroupRefreshWork, "doWork", doWork) config.AutomaticPurging.Enabled = True
def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.recordType: RecordType.user, })) # Add a group records.append( TestRecord( self.directory, { fieldName.uid: u"testgroup", fieldName.recordType: RecordType.group, })) yield self.directory.updateRecords(records, create=True) group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members)
def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.recordType: RecordType.user, } ) ) # Add a group records.append( TestRecord( self.directory, { fieldName.uid: u"testgroup", fieldName.recordType: RecordType.group, } ) ) yield self.directory.updateRecords(records, create=True) group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members)
def setUp(self): self.numUsers = 1000 # The "local" directory service self.directory = DirectoryService(None) # The "remote" directory service remoteDirectory = CalendarInMemoryDirectoryService(None) # Add users records = [] fieldName = remoteDirectory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( remoteDirectory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.recordType: RecordType.user, } ) ) # Add a big group records.append( TestRecord( remoteDirectory, { fieldName.uid: u"bigGroup", fieldName.recordType: RecordType.group, } ) ) yield remoteDirectory.updateRecords(records, create=True) group = yield remoteDirectory.recordWithUID(u"bigGroup") members = yield remoteDirectory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) # Connect the two services directly via an IOPump client = AMP() self.server = DirectoryProxyAMPProtocol(remoteDirectory) pump = returnConnected(self.server, client) # Replace the normal _getConnection method with one that bypasses any # actual networking self.patch(self.directory, "_getConnection", lambda: succeed(client)) # Wrap the normal _call method with one that flushes the IOPump # afterwards origCall = self.directory._call def newCall(*args, **kwds): d = origCall(*args, **kwds) pump.flush() return d self.patch(self.directory, "_call", newCall)
def buildDirectory(store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo, serversDB=None): """ Return a directory without using a config object; suitable for tests which need to have mulitple directory instances. @param store: The store. @param dataRoot: The path to the directory containing xml files for any xml based services. @param servicesInfo: An interable of ConfigDicts mirroring the DirectoryService and ResourceService sections of stdconfig @param augmentServiceInfo: A ConfigDict mirroring the AugmentService section of stdconfig @param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig @param serversDB: A ServersDB object to assign to the directory """ aggregatedServices = [] for serviceValue in servicesInfo: if not serviceValue.Enabled: continue directoryType = serviceValue.type.lower() params = serviceValue.params if "xml" in directoryType: xmlFile = params.xmlFile xmlFile = fullServerPath(dataRoot, xmlFile) fp = FilePath(xmlFile) if not fp.exists(): fp.setContent(DEFAULT_XML_CONTENT) directory = XMLDirectoryService(fp) elif "opendirectory" in directoryType: from txdav.who.opendirectory import (DirectoryService as ODDirectoryService) # We don't want system accounts returned in lookups, so tell # the service to suppress them. directory = ODDirectoryService(suppressSystemRecords=True) elif "ldap" in directoryType: if params.credentials.dn and params.credentials.password: creds = UsernamePassword(params.credentials.dn, params.credentials.password) else: creds = None directory = LDAPDirectoryService( params.uri, params.rdnSchema.base, credentials=creds, fieldNameToAttributesMap=MappingProxyType({ BaseFieldName.uid: ("apple-generateduid", ), BaseFieldName.guid: ("apple-generateduid", ), BaseFieldName.shortNames: (LDAPAttribute.uid.value, ), BaseFieldName.fullNames: (LDAPAttribute.cn.value, ), BaseFieldName.emailAddresses: (LDAPAttribute.mail.value, ), BaseFieldName.password: (LDAPAttribute.userPassword.value, ), LDAPFieldName.memberDNs: (LDAPAttribute.uniqueMember.value, ), }), recordTypeSchemas=MappingProxyType({ RecordType.user: RecordTypeSchema( relativeDN=u"ou=People", # (objectClass=inetOrgPerson) attributes=(( LDAPAttribute.objectClass.value, LDAPObjectClass.inetOrgPerson.value, ), ), ), RecordType.group: RecordTypeSchema( relativeDN=u"ou=Groups", # (objectClass=groupOfNames) attributes=(( LDAPAttribute.objectClass.value, LDAPObjectClass.groupOfUniqueNames.value, ), ), ), })) elif "inmemory" in directoryType: from txdav.who.test.support import CalendarInMemoryDirectoryService directory = CalendarInMemoryDirectoryService() else: log.error("Invalid DirectoryType: {dt}", dt=directoryType) raise DirectoryConfigurationError # Set the appropriate record types on each service types = [] fieldNames = [] for recordTypeName in params.recordTypes: recordType = { "users": RecordType.user, "groups": RecordType.group, "locations": CalRecordType.location, "resources": CalRecordType.resource, "addresses": CalRecordType.address, }.get(recordTypeName, None) if recordType is None: log.error("Invalid Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError if recordType in types: log.error("Duplicate Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError types.append(recordType) directory.recordType = ConstantsContainer(types) directory.fieldName = ConstantsContainer( (directory.fieldName, CalFieldName)) fieldNames.append(directory.fieldName) aggregatedServices.append(directory) # # Setup the Augment Service # if augmentServiceInfo.type: for augmentFile in augmentServiceInfo.params.xmlFiles: augmentFile = fullServerPath(dataRoot, augmentFile) augmentFilePath = FilePath(augmentFile) if not augmentFilePath.exists(): augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT) augmentClass = namedClass(augmentServiceInfo.type) log.info("Configuring augment service of type: {augmentClass}", augmentClass=augmentClass) try: augmentService = augmentClass(**augmentServiceInfo.params) except IOError: log.error("Could not start augment service") raise else: augmentService = None userDirectory = None for directory in aggregatedServices: if RecordType.user in directory.recordTypes(): userDirectory = directory break else: log.error("No directory service set up for users") raise DirectoryConfigurationError # Delegate service delegateDirectory = DelegateDirectoryService(userDirectory.realmName, store) aggregatedServices.append(delegateDirectory) # Wiki service if wikiServiceInfo.Enabled: aggregatedServices.append( WikiDirectoryService(userDirectory.realmName, wikiServiceInfo.CollabHost, wikiServiceInfo.CollabPort)) # Aggregate service aggregateDirectory = AggregateDirectoryService(userDirectory.realmName, aggregatedServices) # Augment service try: fieldNames.append(CalFieldName) augmented = AugmentedDirectoryService(aggregateDirectory, store, augmentService) augmented.fieldName = ConstantsContainer(fieldNames) # The delegate directory needs a way to look up user/group records # so hand it a reference to the augmented directory. # FIXME: is there a better pattern to use here? delegateDirectory.setMasterDirectory(augmented) except Exception as e: log.error("Could not create directory service", error=e) raise if serversDB is not None: augmented.setServersDB(serversDB) return augmented
class DynamicGroupTest(StoreTestCase): @inlineCallbacks def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.recordType: RecordType.user, })) # Add two groups for uid in ( u"testgroup", u"emptygroup", ): records.append( TestRecord(self.directory, { fieldName.uid: uid, fieldName.recordType: RecordType.group, })) yield self.directory.updateRecords(records, create=True) # add members to test group group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) def doWork(self): self.transaction._groupCacher = groupCacher return unpatchedDoWork(self) groupCacher = self.groupCacher unpatchedDoWork = GroupRefreshWork.doWork self.patch(GroupRefreshWork, "doWork", doWork) config.AutomaticPurging.Enabled = True @inlineCallbacks def test_extant(self): """ Verify that once a group is removed from the directory, the next call to refreshGroup() will set the "extent" to False. Add the group back to the directory and "extent" becomes True. """ store = self.storeUnderTest() for uid in ( u"testgroup", u"emptygroup", ): txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = yield txn.groupByUID(uid) yield txn.commit() self.assertTrue(group.extant) # Remove the group yield self.directory.removeRecords([uid]) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = (yield txn.groupByUID(uid)) yield txn.commit() # Extant = False self.assertFalse(group.extant) # The list of members stored in the DB for this group is now empty txn = store.newTransaction() members = yield txn.groupMemberUIDs(group.groupID) yield txn.commit() self.assertEquals(members, set()) # Add the group back into the directory fieldName = self.directory.fieldName yield self.directory.updateRecords( (TestRecord(self.directory, { fieldName.uid: uid, fieldName.recordType: RecordType.group, }), ), create=True) if uid == u"testgroup": group = yield self.directory.recordWithUID(uid) members = yield self.directory.recordsWithRecordType( RecordType.user) yield group.setMembers(members) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = (yield txn.groupByUID(uid)) yield txn.commit() # Extant = True self.assertTrue(group.extant) # The list of members stored in the DB for this group has 100 users txn = store.newTransaction() members = yield txn.groupMemberUIDs(group.groupID) yield txn.commit() self.assertEquals(len(members), 100 if uid == u"testgroup" else 0) @inlineCallbacks def test_update_delete_unused(self): """ Verify that unused groups are deleted from group cache """ store = self.storeUnderTest() # unused group deleted for uid in ( u"testgroup", u"emptygroup", ): txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) txn = store.newTransaction() yield self.groupCacher.update(txn) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertEqual(group, None) # delegate groups not deleted for uid in ( u"testgroup", u"emptygroup", ): txn = store.newTransaction() group = yield txn.groupByUID(uid) yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True) yield txn.commit() self.assertNotEqual(group, None) txn = store.newTransaction() yield self.groupCacher.update(txn) yield txn.commit() yield JobItem.waitEmpty(store.newTransaction, reactor, 60) txn = store.newTransaction() group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) # delegate group is deleted. unused group is deleted txn = store.newTransaction() testGroup = yield txn.groupByUID(u"testgroup", create=False) yield txn.removeDelegateGroup(delegator=u"sagen", delegateGroupID=testGroup.groupID, readWrite=True) testGroup = yield txn.groupByUID(u"testgroup", create=False) emptyGroup = yield txn.groupByUID(u"emptygroup", create=False) yield txn.commit() self.assertNotEqual(testGroup, None) self.assertNotEqual(emptyGroup, None) txn = store.newTransaction() yield self.groupCacher.update(txn) yield txn.commit() yield JobItem.waitEmpty(store.newTransaction, reactor, 60) txn = store.newTransaction() testGroup = yield txn.groupByUID(u"testgroup", create=False) emptyGroup = yield txn.groupByUID(u"emptygroup", create=False) yield txn.commit() self.assertEqual(testGroup, None) self.assertNotEqual(emptyGroup, None) @inlineCallbacks def test_update_delete_old_nonextant(self): """ Verify that old missing groups are deleted from group cache """ oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds store = self.storeUnderTest() for uid in ( u"testgroup", u"emptygroup", ): config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds txn = store.newTransaction() group = yield txn.groupByUID(uid) yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) self.assertTrue(group.extant) # Remove the group, still cached yield self.directory.removeRecords([uid]) txn = store.newTransaction() yield self.groupCacher.update(txn) group = yield txn.groupByUID(uid, create=False) yield txn.commit() yield JobItem.waitEmpty(store.newTransaction, reactor, 60) txn = store.newTransaction() group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) self.assertFalse(group.extant) # delete the group config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0" txn = store.newTransaction() yield self.groupCacher.update(txn) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertEqual(group, None)
class DynamicGroupTest(StoreTestCase): @inlineCallbacks def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ), fieldName.recordType: RecordType.user, })) # Add a group records.append( TestRecord( self.directory, { fieldName.uid: u"testgroup", fieldName.recordType: RecordType.group, })) yield self.directory.updateRecords(records, create=True) group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) @inlineCallbacks def test_extant(self): """ Verify that once a group is removed from the directory, the next call to refreshGroup() will set the "extent" to False. Add the group back to the directory and "extent" becomes True. """ store = self.storeUnderTest() txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, u"testgroup") (groupID, _ignore_name, membershipHash, _ignore_modified, extant) = (yield txn.groupByUID(u"testgroup")) yield txn.commit() self.assertTrue(extant) # Remove the group yield self.directory.removeRecords([u"testgroup"]) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, u"testgroup") (groupID, _ignore_name, membershipHash, _ignore_modified, extant) = (yield txn.groupByUID(u"testgroup")) yield txn.commit() # Extant = False self.assertFalse(extant) # The list of members stored in the DB for this group is now empty txn = store.newTransaction() members = yield txn.membersOfGroup(groupID) yield txn.commit() self.assertEquals(members, set()) # Add the group back into the directory fieldName = self.directory.fieldName yield self.directory.updateRecords((TestRecord( self.directory, { fieldName.uid: u"testgroup", fieldName.recordType: RecordType.group, }), ), create=True) group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, u"testgroup") (groupID, _ignore_name, membershipHash, _ignore_modified, extant) = (yield txn.groupByUID(u"testgroup")) yield txn.commit() # Extant = True self.assertTrue(extant) # The list of members stored in the DB for this group has 100 users txn = store.newTransaction() members = yield txn.membersOfGroup(groupID) yield txn.commit() self.assertEquals(len(members), 100)
def buildDirectory( store, dataRoot, servicesInfo, augmentServiceInfo, wikiServiceInfo, serversDB=None, cachingSeconds=0, filterStartsWith=False, lookupsBetweenPurges=0, negativeCaching=True, ): """ Return a directory without using a config object; suitable for tests which need to have mulitple directory instances. @param store: The store. @param dataRoot: The path to the directory containing xml files for any xml based services. @param servicesInfo: An interable of ConfigDicts mirroring the DirectoryService and ResourceService sections of stdconfig @param augmentServiceInfo: A ConfigDict mirroring the AugmentService section of stdconfig @param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig @param serversDB: A ServersDB object to assign to the directory """ aggregatedServices = [] cachingServices = [] ldapService = None # LDAP DS has extra stats (see augment.py) for serviceValue in servicesInfo: if not serviceValue.Enabled: continue directoryType = serviceValue.type.lower() params = serviceValue.params if "xml" in directoryType: xmlFile = params.xmlFile xmlFile = fullServerPath(dataRoot, xmlFile) fp = FilePath(xmlFile) if not fp.exists(): fp.setContent(DEFAULT_XML_CONTENT) directory = XMLDirectoryService(fp) elif "opendirectory" in directoryType: from txdav.who.opendirectory import (DirectoryService as ODDirectoryService) # We don't want system accounts returned in lookups, so tell # the service to suppress them. node = params.node directory = ODDirectoryService(nodeName=node, suppressSystemRecords=True) elif "ldap" in directoryType: from twext.who.ldap import (DirectoryService as LDAPDirectoryService, FieldName as LDAPFieldName, RecordTypeSchema) if params.credentials.dn and params.credentials.password: creds = UsernamePassword(params.credentials.dn, params.credentials.password) else: creds = None mapping = params.mapping extraFilters = params.extraFilters directory = LDAPDirectoryService( params.uri, params.rdnSchema.base, useTLS=params.useTLS, credentials=creds, fieldNameToAttributesMap=MappingProxyType({ BaseFieldName.uid: mapping.uid, BaseFieldName.guid: mapping.guid, BaseFieldName.shortNames: mapping.shortNames, BaseFieldName.fullNames: mapping.fullNames, BaseFieldName.emailAddresses: mapping.emailAddresses, LDAPFieldName.memberDNs: mapping.memberDNs, CalFieldName.readOnlyProxy: mapping.readOnlyProxy, CalFieldName.readWriteProxy: mapping.readWriteProxy, CalFieldName.loginAllowed: mapping.loginAllowed, CalFieldName.hasCalendars: mapping.hasCalendars, CalFieldName.autoScheduleMode: mapping.autoScheduleMode, CalFieldName.autoAcceptGroup: mapping.autoAcceptGroup, CalFieldName.serviceNodeUID: mapping.serviceNodeUID, CalFieldName.associatedAddress: mapping.associatedAddress, CalFieldName.geographicLocation: mapping.geographicLocation, CalFieldName.streetAddress: mapping.streetAddress, }), recordTypeSchemas=MappingProxyType({ RecordType.user: RecordTypeSchema( relativeDN=params.rdnSchema.users, attributes=(), ), RecordType.group: RecordTypeSchema( relativeDN=params.rdnSchema.groups, attributes=(), ), CalRecordType.location: RecordTypeSchema( relativeDN=params.rdnSchema.locations, attributes=(), ), CalRecordType.resource: RecordTypeSchema( relativeDN=params.rdnSchema.resources, attributes=(), ), CalRecordType.address: RecordTypeSchema( relativeDN=params.rdnSchema.addresses, attributes=(), ), }), extraFilters={ RecordType.user: extraFilters.get("users", ""), RecordType.group: extraFilters.get("groups", ""), CalRecordType.location: extraFilters.get("locations", ""), CalRecordType.resource: extraFilters.get("resources", ""), CalRecordType.address: extraFilters.get("addresses", ""), }, threadPoolMax=params.get("threadPoolMax", 10), authConnectionMax=params.get("authConnectionMax", 5), queryConnectionMax=params.get("queryConnectionMax", 5), tries=params.get("tries", 3), warningThresholdSeconds=params.get("warningThresholdSeconds", 5), ) ldapService = directory elif "inmemory" in directoryType: from txdav.who.test.support import CalendarInMemoryDirectoryService directory = CalendarInMemoryDirectoryService() else: log.error("Invalid DirectoryType: {dt}", dt=directoryType) raise DirectoryConfigurationError # Set the appropriate record types on each service types = [] fieldNames = [] for recordTypeName in params.recordTypes: recordType = { "users": RecordType.user, "groups": RecordType.group, "locations": CalRecordType.location, "resources": CalRecordType.resource, "addresses": CalRecordType.address, }.get(recordTypeName, None) if recordType is None: log.error("Invalid Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError if recordType in types: log.error("Duplicate Record Type: {rt}", rt=recordTypeName) raise DirectoryConfigurationError types.append(recordType) directory.recordType = ConstantsContainer(types) directory.fieldName = ConstantsContainer( (directory.fieldName, CalFieldName)) fieldNames.append(directory.fieldName) if cachingSeconds: directory = CachingDirectoryService( directory, expireSeconds=cachingSeconds, lookupsBetweenPurges=lookupsBetweenPurges, negativeCaching=negativeCaching, ) cachingServices.append(directory) aggregatedServices.append(directory) # # Setup the Augment Service # serviceClass = { "xml": "twistedcaldav.directory.augment.AugmentXMLDB", } for augmentFile in augmentServiceInfo.params.xmlFiles: augmentFile = fullServerPath(dataRoot, augmentFile) augmentFilePath = FilePath(augmentFile) if not augmentFilePath.exists(): augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT) augmentClass = namedClass(serviceClass[augmentServiceInfo.type]) log.info("Configuring augment service of type: {augmentClass}", augmentClass=augmentClass) try: augmentService = augmentClass(**augmentServiceInfo.params) except IOError: log.error("Could not start augment service") raise userDirectory = None for directory in aggregatedServices: if RecordType.user in directory.recordTypes(): userDirectory = directory break else: log.error("No directory service set up for users") raise DirectoryConfigurationError # Delegate service delegateDirectory = DelegateDirectoryService(userDirectory.realmName, store) # (put at front of list so we don't try to ask the actual DS services # about the delegate-related principals, for performance) aggregatedServices.insert(0, delegateDirectory) # Wiki service if wikiServiceInfo.Enabled: aggregatedServices.append( WikiDirectoryService( userDirectory.realmName, wikiServiceInfo.EndpointDescriptor, )) # Aggregate service aggregateDirectory = AggregateDirectoryService(userDirectory.realmName, aggregatedServices) # Augment service try: fieldNames.append(CalFieldName) augmented = AugmentedDirectoryService(aggregateDirectory, store, augmentService) augmented.fieldName = ConstantsContainer(fieldNames) # The delegate directory needs a way to look up user/group records # so hand it a reference to the augmented directory. # FIXME: is there a better pattern to use here? delegateDirectory.setMasterDirectory(augmented) # Tell each caching service what method to use when reporting # times and cache stats for cachingService in cachingServices: cachingService.setTimingMethod(augmented._addTiming) # LDAP has additional stats to report augmented._ldapDS = ldapService except Exception as e: log.error("Could not create directory service", error=e) raise if serversDB is not None: augmented.setServersDB(serversDB) if filterStartsWith: augmented.setFilter(startswithFilter) return augmented
class DynamicGroupTest(StoreTestCase): @inlineCallbacks def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.recordType: RecordType.user, } ) ) # Add a group records.append( TestRecord( self.directory, { fieldName.uid: u"testgroup", fieldName.recordType: RecordType.group, } ) ) yield self.directory.updateRecords(records, create=True) group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) @inlineCallbacks def test_extant(self): """ Verify that once a group is removed from the directory, the next call to refreshGroup() will set the "extent" to False. Add the group back to the directory and "extent" becomes True. """ store = self.storeUnderTest() txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, u"testgroup") ( groupID, _ignore_name, membershipHash, _ignore_modified, extant ) = (yield txn.groupByUID(u"testgroup")) yield txn.commit() self.assertTrue(extant) # Remove the group yield self.directory.removeRecords([u"testgroup"]) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, u"testgroup") ( groupID, _ignore_name, membershipHash, _ignore_modified, extant ) = (yield txn.groupByUID(u"testgroup")) yield txn.commit() # Extant = False self.assertFalse(extant) # The list of members stored in the DB for this group is now empty txn = store.newTransaction() members = yield txn.membersOfGroup(groupID) yield txn.commit() self.assertEquals(members, set()) # Add the group back into the directory fieldName = self.directory.fieldName yield self.directory.updateRecords( ( TestRecord( self.directory, { fieldName.uid: u"testgroup", fieldName.recordType: RecordType.group, } ), ), create=True ) group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, u"testgroup") ( groupID, _ignore_name, membershipHash, _ignore_modified, extant ) = (yield txn.groupByUID(u"testgroup")) yield txn.commit() # Extant = True self.assertTrue(extant) # The list of members stored in the DB for this group has 100 users txn = store.newTransaction() members = yield txn.membersOfGroup(groupID) yield txn.commit() self.assertEquals(len(members), 100)
class DynamicGroupTest(StoreTestCase): @inlineCallbacks def setUp(self): yield super(DynamicGroupTest, self).setUp() self.directory = CalendarInMemoryDirectoryService(None) self.store.setDirectoryService(self.directory) self.groupCacher = GroupCacher(self.directory) self.numUsers = 100 # Add users records = [] fieldName = self.directory.fieldName for i in xrange(self.numUsers): records.append( TestRecord( self.directory, { fieldName.uid: u"foo{ctr:05d}".format(ctr=i), fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),), fieldName.recordType: RecordType.user, } ) ) # Add two groups for uid in (u"testgroup", u"emptygroup",): records.append( TestRecord( self.directory, { fieldName.uid: uid, fieldName.recordType: RecordType.group, } ) ) yield self.directory.updateRecords(records, create=True) # add members to test group group = yield self.directory.recordWithUID(u"testgroup") members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) def doWork(self): self.transaction._groupCacher = groupCacher return unpatchedDoWork(self) groupCacher = self.groupCacher unpatchedDoWork = GroupRefreshWork.doWork self.patch(GroupRefreshWork, "doWork", doWork) config.AutomaticPurging.Enabled = True @inlineCallbacks def test_extant(self): """ Verify that once a group is removed from the directory, the next call to refreshGroup() will set the "extent" to False. Add the group back to the directory and "extent" becomes True. """ store = self.storeUnderTest() for uid in (u"testgroup", u"emptygroup",): txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = yield txn.groupByUID(uid) yield txn.commit() self.assertTrue(group.extant) # Remove the group yield self.directory.removeRecords([uid]) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = (yield txn.groupByUID(uid)) yield txn.commit() # Extant = False self.assertFalse(group.extant) # The list of members stored in the DB for this group is now empty txn = store.newTransaction() members = yield txn.groupMemberUIDs(group.groupID) yield txn.commit() self.assertEquals(members, set()) # Add the group back into the directory fieldName = self.directory.fieldName yield self.directory.updateRecords( ( TestRecord( self.directory, { fieldName.uid: uid, fieldName.recordType: RecordType.group, } ), ), create=True ) if uid == u"testgroup": group = yield self.directory.recordWithUID(uid) members = yield self.directory.recordsWithRecordType(RecordType.user) yield group.setMembers(members) txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = (yield txn.groupByUID(uid)) yield txn.commit() # Extant = True self.assertTrue(group.extant) # The list of members stored in the DB for this group has 100 users txn = store.newTransaction() members = yield txn.groupMemberUIDs(group.groupID) yield txn.commit() self.assertEquals(len(members), 100 if uid == u"testgroup" else 0) @inlineCallbacks def test_update_delete_unused(self): """ Verify that unused groups are deleted from group cache """ store = self.storeUnderTest() # unused group deleted for uid in (u"testgroup", u"emptygroup",): txn = store.newTransaction() yield self.groupCacher.refreshGroup(txn, uid) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) txn = store.newTransaction() yield self.groupCacher.update(txn) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertEqual(group, None) # delegate groups not deleted for uid in (u"testgroup", u"emptygroup",): txn = store.newTransaction() group = yield txn.groupByUID(uid) yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True) yield txn.commit() self.assertNotEqual(group, None) txn = store.newTransaction() yield self.groupCacher.update(txn) yield txn.commit() yield JobItem.waitEmpty(store.newTransaction, reactor, 60) txn = store.newTransaction() group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) # delegate group is deleted. unused group is deleted txn = store.newTransaction() testGroup = yield txn.groupByUID(u"testgroup", create=False) yield txn.removeDelegateGroup(delegator=u"sagen", delegateGroupID=testGroup.groupID, readWrite=True) testGroup = yield txn.groupByUID(u"testgroup", create=False) emptyGroup = yield txn.groupByUID(u"emptygroup", create=False) yield txn.commit() self.assertNotEqual(testGroup, None) self.assertNotEqual(emptyGroup, None) txn = store.newTransaction() yield self.groupCacher.update(txn) yield txn.commit() yield JobItem.waitEmpty(store.newTransaction, reactor, 60) txn = store.newTransaction() testGroup = yield txn.groupByUID(u"testgroup", create=False) emptyGroup = yield txn.groupByUID(u"emptygroup", create=False) yield txn.commit() self.assertEqual(testGroup, None) self.assertNotEqual(emptyGroup, None) @inlineCallbacks def test_update_delete_old_nonextant(self): """ Verify that old missing groups are deleted from group cache """ oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds store = self.storeUnderTest() for uid in (u"testgroup", u"emptygroup",): config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds txn = store.newTransaction() group = yield txn.groupByUID(uid) yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) self.assertTrue(group.extant) # Remove the group, still cached yield self.directory.removeRecords([uid]) txn = store.newTransaction() yield self.groupCacher.update(txn) group = yield txn.groupByUID(uid, create=False) yield txn.commit() yield JobItem.waitEmpty(store.newTransaction, reactor, 60) txn = store.newTransaction() group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertNotEqual(group, None) self.assertFalse(group.extant) # delete the group config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0" txn = store.newTransaction() yield self.groupCacher.update(txn) group = yield txn.groupByUID(uid, create=False) yield txn.commit() self.assertEqual(group, None)