def setUp(self):
self.numUsers = 1000
# The "local" directory service
self.directory = DirectoryService(None)
# The "remote" directory service
remoteDirectory = CalendarInMemoryDirectoryService(None)
# Add users
records = []
fieldName = remoteDirectory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
remoteDirectory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add a big group
records.append(
TestRecord(
remoteDirectory, {
fieldName.uid: u"bigGroup",
fieldName.recordType: RecordType.group,
}))
yield remoteDirectory.updateRecords(records, create=True)
group = yield remoteDirectory.recordWithUID(u"bigGroup")
members = yield remoteDirectory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
# Connect the two services directly via an IOPump
client = AMP()
self.server = DirectoryProxyAMPProtocol(remoteDirectory)
pump = returnConnected(self.server, client)
# Replace the normal _getConnection method with one that bypasses any
# actual networking
self.patch(self.directory, "_getConnection", lambda: succeed(client))
# Wrap the normal _call method with one that flushes the IOPump
# afterwards
origCall = self.directory._call
def newCall(*args, **kwds):
d = origCall(*args, **kwds)
pump.flush()
return d
self.patch(self.directory, "_call", newCall)
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add two groups
for uid in (u"testgroup", u"emptygroup",):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
# add members to test group
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def doWork(self):
self.transaction._groupCacher = groupCacher
return unpatchedDoWork(self)
groupCacher = self.groupCacher
unpatchedDoWork = GroupRefreshWork.doWork
self.patch(GroupRefreshWork, "doWork", doWork)
config.AutomaticPurging.Enabled = True
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add two groups
for uid in (u"testgroup", u"emptygroup",):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
# add members to test group
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def doWork(self):
self.transaction._groupCacher = groupCacher
return unpatchedDoWork(self)
groupCacher = self.groupCacher
unpatchedDoWork = GroupRefreshWork.doWork
self.patch(GroupRefreshWork, "doWork", doWork)
config.AutomaticPurging.Enabled = True
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add a group
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}))
yield self.directory.updateRecords(records, create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add a group
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def setUp(self):
self.numUsers = 1000
# The "local" directory service
self.directory = DirectoryService(None)
# The "remote" directory service
remoteDirectory = CalendarInMemoryDirectoryService(None)
# Add users
records = []
fieldName = remoteDirectory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
remoteDirectory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add a big group
records.append(
TestRecord(
remoteDirectory,
{
fieldName.uid: u"bigGroup",
fieldName.recordType: RecordType.group,
}
)
)
yield remoteDirectory.updateRecords(records, create=True)
group = yield remoteDirectory.recordWithUID(u"bigGroup")
members = yield remoteDirectory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
# Connect the two services directly via an IOPump
client = AMP()
self.server = DirectoryProxyAMPProtocol(remoteDirectory)
pump = returnConnected(self.server, client)
# Replace the normal _getConnection method with one that bypasses any
# actual networking
self.patch(self.directory, "_getConnection", lambda: succeed(client))
# Wrap the normal _call method with one that flushes the IOPump
# afterwards
origCall = self.directory._call
def newCall(*args, **kwds):
d = origCall(*args, **kwds)
pump.flush()
return d
self.patch(self.directory, "_call", newCall)
def buildDirectory(store,
dataRoot,
servicesInfo,
augmentServiceInfo,
wikiServiceInfo,
serversDB=None):
"""
Return a directory without using a config object; suitable for tests
which need to have mulitple directory instances.
@param store: The store.
@param dataRoot: The path to the directory containing xml files for any xml
based services.
@param servicesInfo: An interable of ConfigDicts mirroring the
DirectoryService and ResourceService sections of stdconfig
@param augmentServiceInfo: A ConfigDict mirroring the AugmentService section
of stdconfig
@param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig
@param serversDB: A ServersDB object to assign to the directory
"""
aggregatedServices = []
for serviceValue in servicesInfo:
if not serviceValue.Enabled:
continue
directoryType = serviceValue.type.lower()
params = serviceValue.params
if "xml" in directoryType:
xmlFile = params.xmlFile
xmlFile = fullServerPath(dataRoot, xmlFile)
fp = FilePath(xmlFile)
if not fp.exists():
fp.setContent(DEFAULT_XML_CONTENT)
directory = XMLDirectoryService(fp)
elif "opendirectory" in directoryType:
from txdav.who.opendirectory import (DirectoryService as
ODDirectoryService)
# We don't want system accounts returned in lookups, so tell
# the service to suppress them.
directory = ODDirectoryService(suppressSystemRecords=True)
elif "ldap" in directoryType:
if params.credentials.dn and params.credentials.password:
creds = UsernamePassword(params.credentials.dn,
params.credentials.password)
else:
creds = None
directory = LDAPDirectoryService(
params.uri,
params.rdnSchema.base,
credentials=creds,
fieldNameToAttributesMap=MappingProxyType({
BaseFieldName.uid: ("apple-generateduid", ),
BaseFieldName.guid: ("apple-generateduid", ),
BaseFieldName.shortNames: (LDAPAttribute.uid.value, ),
BaseFieldName.fullNames: (LDAPAttribute.cn.value, ),
BaseFieldName.emailAddresses: (LDAPAttribute.mail.value, ),
BaseFieldName.password:
(LDAPAttribute.userPassword.value, ),
LDAPFieldName.memberDNs:
(LDAPAttribute.uniqueMember.value, ),
}),
recordTypeSchemas=MappingProxyType({
RecordType.user:
RecordTypeSchema(
relativeDN=u"ou=People",
# (objectClass=inetOrgPerson)
attributes=((
LDAPAttribute.objectClass.value,
LDAPObjectClass.inetOrgPerson.value,
), ),
),
RecordType.group:
RecordTypeSchema(
relativeDN=u"ou=Groups",
# (objectClass=groupOfNames)
attributes=((
LDAPAttribute.objectClass.value,
LDAPObjectClass.groupOfUniqueNames.value,
), ),
),
}))
elif "inmemory" in directoryType:
from txdav.who.test.support import CalendarInMemoryDirectoryService
directory = CalendarInMemoryDirectoryService()
else:
log.error("Invalid DirectoryType: {dt}", dt=directoryType)
raise DirectoryConfigurationError
# Set the appropriate record types on each service
types = []
fieldNames = []
for recordTypeName in params.recordTypes:
recordType = {
"users": RecordType.user,
"groups": RecordType.group,
"locations": CalRecordType.location,
"resources": CalRecordType.resource,
"addresses": CalRecordType.address,
}.get(recordTypeName, None)
if recordType is None:
log.error("Invalid Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
if recordType in types:
log.error("Duplicate Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
types.append(recordType)
directory.recordType = ConstantsContainer(types)
directory.fieldName = ConstantsContainer(
(directory.fieldName, CalFieldName))
fieldNames.append(directory.fieldName)
aggregatedServices.append(directory)
#
# Setup the Augment Service
#
if augmentServiceInfo.type:
for augmentFile in augmentServiceInfo.params.xmlFiles:
augmentFile = fullServerPath(dataRoot, augmentFile)
augmentFilePath = FilePath(augmentFile)
if not augmentFilePath.exists():
augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT)
augmentClass = namedClass(augmentServiceInfo.type)
log.info("Configuring augment service of type: {augmentClass}",
augmentClass=augmentClass)
try:
augmentService = augmentClass(**augmentServiceInfo.params)
except IOError:
log.error("Could not start augment service")
raise
else:
augmentService = None
userDirectory = None
for directory in aggregatedServices:
if RecordType.user in directory.recordTypes():
userDirectory = directory
break
else:
log.error("No directory service set up for users")
raise DirectoryConfigurationError
# Delegate service
delegateDirectory = DelegateDirectoryService(userDirectory.realmName,
store)
aggregatedServices.append(delegateDirectory)
# Wiki service
if wikiServiceInfo.Enabled:
aggregatedServices.append(
WikiDirectoryService(userDirectory.realmName,
wikiServiceInfo.CollabHost,
wikiServiceInfo.CollabPort))
# Aggregate service
aggregateDirectory = AggregateDirectoryService(userDirectory.realmName,
aggregatedServices)
# Augment service
try:
fieldNames.append(CalFieldName)
augmented = AugmentedDirectoryService(aggregateDirectory, store,
augmentService)
augmented.fieldName = ConstantsContainer(fieldNames)
# The delegate directory needs a way to look up user/group records
# so hand it a reference to the augmented directory.
# FIXME: is there a better pattern to use here?
delegateDirectory.setMasterDirectory(augmented)
except Exception as e:
log.error("Could not create directory service", error=e)
raise
if serversDB is not None:
augmented.setServersDB(serversDB)
return augmented
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add two groups
for uid in (
u"testgroup",
u"emptygroup",
):
records.append(
TestRecord(self.directory, {
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}))
yield self.directory.updateRecords(records, create=True)
# add members to test group
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def doWork(self):
self.transaction._groupCacher = groupCacher
return unpatchedDoWork(self)
groupCacher = self.groupCacher
unpatchedDoWork = GroupRefreshWork.doWork
self.patch(GroupRefreshWork, "doWork", doWork)
config.AutomaticPurging.Enabled = True
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
for uid in (
u"testgroup",
u"emptygroup",
):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid)
yield txn.commit()
self.assertTrue(group.extant)
# Remove the group
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = False
self.assertFalse(group.extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords(
(TestRecord(self.directory, {
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}), ),
create=True)
if uid == u"testgroup":
group = yield self.directory.recordWithUID(uid)
members = yield self.directory.recordsWithRecordType(
RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = True
self.assertTrue(group.extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(len(members), 100 if uid == u"testgroup" else 0)
@inlineCallbacks
def test_update_delete_unused(self):
"""
Verify that unused groups are deleted from group cache
"""
store = self.storeUnderTest()
# unused group deleted
for uid in (
u"testgroup",
u"emptygroup",
):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
# delegate groups not deleted
for uid in (
u"testgroup",
u"emptygroup",
):
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen",
delegateGroupID=group.groupID,
readWrite=True)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
# delegate group is deleted. unused group is deleted
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
yield txn.removeDelegateGroup(delegator=u"sagen",
delegateGroupID=testGroup.groupID,
readWrite=True)
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertNotEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
@inlineCallbacks
def test_update_delete_old_nonextant(self):
"""
Verify that old missing groups are deleted from group cache
"""
oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds
store = self.storeUnderTest()
for uid in (
u"testgroup",
u"emptygroup",
):
config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen",
delegateGroupID=group.groupID,
readWrite=True)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertTrue(group.extant)
# Remove the group, still cached
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertFalse(group.extant)
# delete the group
config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0"
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add a group
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}))
yield self.directory.updateRecords(records, create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(groupID, _ignore_name, membershipHash, _ignore_modified,
extant) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
self.assertTrue(extant)
# Remove the group
yield self.directory.removeRecords([u"testgroup"])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(groupID, _ignore_name, membershipHash, _ignore_modified,
extant) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = False
self.assertFalse(extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords((TestRecord(
self.directory, {
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}), ),
create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(groupID, _ignore_name, membershipHash, _ignore_modified,
extant) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = True
self.assertTrue(extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(len(members), 100)
def buildDirectory(
store,
dataRoot,
servicesInfo,
augmentServiceInfo,
wikiServiceInfo,
serversDB=None,
cachingSeconds=0,
filterStartsWith=False,
lookupsBetweenPurges=0,
negativeCaching=True,
):
"""
Return a directory without using a config object; suitable for tests
which need to have mulitple directory instances.
@param store: The store.
@param dataRoot: The path to the directory containing xml files for any xml
based services.
@param servicesInfo: An interable of ConfigDicts mirroring the
DirectoryService and ResourceService sections of stdconfig
@param augmentServiceInfo: A ConfigDict mirroring the AugmentService section
of stdconfig
@param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig
@param serversDB: A ServersDB object to assign to the directory
"""
aggregatedServices = []
cachingServices = []
ldapService = None # LDAP DS has extra stats (see augment.py)
for serviceValue in servicesInfo:
if not serviceValue.Enabled:
continue
directoryType = serviceValue.type.lower()
params = serviceValue.params
if "xml" in directoryType:
xmlFile = params.xmlFile
xmlFile = fullServerPath(dataRoot, xmlFile)
fp = FilePath(xmlFile)
if not fp.exists():
fp.setContent(DEFAULT_XML_CONTENT)
directory = XMLDirectoryService(fp)
elif "opendirectory" in directoryType:
from txdav.who.opendirectory import (DirectoryService as
ODDirectoryService)
# We don't want system accounts returned in lookups, so tell
# the service to suppress them.
node = params.node
directory = ODDirectoryService(nodeName=node,
suppressSystemRecords=True)
elif "ldap" in directoryType:
from twext.who.ldap import (DirectoryService as
LDAPDirectoryService, FieldName as
LDAPFieldName, RecordTypeSchema)
if params.credentials.dn and params.credentials.password:
creds = UsernamePassword(params.credentials.dn,
params.credentials.password)
else:
creds = None
mapping = params.mapping
extraFilters = params.extraFilters
directory = LDAPDirectoryService(
params.uri,
params.rdnSchema.base,
useTLS=params.useTLS,
credentials=creds,
fieldNameToAttributesMap=MappingProxyType({
BaseFieldName.uid:
mapping.uid,
BaseFieldName.guid:
mapping.guid,
BaseFieldName.shortNames:
mapping.shortNames,
BaseFieldName.fullNames:
mapping.fullNames,
BaseFieldName.emailAddresses:
mapping.emailAddresses,
LDAPFieldName.memberDNs:
mapping.memberDNs,
CalFieldName.readOnlyProxy:
mapping.readOnlyProxy,
CalFieldName.readWriteProxy:
mapping.readWriteProxy,
CalFieldName.loginAllowed:
mapping.loginAllowed,
CalFieldName.hasCalendars:
mapping.hasCalendars,
CalFieldName.autoScheduleMode:
mapping.autoScheduleMode,
CalFieldName.autoAcceptGroup:
mapping.autoAcceptGroup,
CalFieldName.serviceNodeUID:
mapping.serviceNodeUID,
CalFieldName.associatedAddress:
mapping.associatedAddress,
CalFieldName.geographicLocation:
mapping.geographicLocation,
CalFieldName.streetAddress:
mapping.streetAddress,
}),
recordTypeSchemas=MappingProxyType({
RecordType.user:
RecordTypeSchema(
relativeDN=params.rdnSchema.users,
attributes=(),
),
RecordType.group:
RecordTypeSchema(
relativeDN=params.rdnSchema.groups,
attributes=(),
),
CalRecordType.location:
RecordTypeSchema(
relativeDN=params.rdnSchema.locations,
attributes=(),
),
CalRecordType.resource:
RecordTypeSchema(
relativeDN=params.rdnSchema.resources,
attributes=(),
),
CalRecordType.address:
RecordTypeSchema(
relativeDN=params.rdnSchema.addresses,
attributes=(),
),
}),
extraFilters={
RecordType.user: extraFilters.get("users", ""),
RecordType.group: extraFilters.get("groups", ""),
CalRecordType.location: extraFilters.get("locations", ""),
CalRecordType.resource: extraFilters.get("resources", ""),
CalRecordType.address: extraFilters.get("addresses", ""),
},
threadPoolMax=params.get("threadPoolMax", 10),
authConnectionMax=params.get("authConnectionMax", 5),
queryConnectionMax=params.get("queryConnectionMax", 5),
tries=params.get("tries", 3),
warningThresholdSeconds=params.get("warningThresholdSeconds",
5),
)
ldapService = directory
elif "inmemory" in directoryType:
from txdav.who.test.support import CalendarInMemoryDirectoryService
directory = CalendarInMemoryDirectoryService()
else:
log.error("Invalid DirectoryType: {dt}", dt=directoryType)
raise DirectoryConfigurationError
# Set the appropriate record types on each service
types = []
fieldNames = []
for recordTypeName in params.recordTypes:
recordType = {
"users": RecordType.user,
"groups": RecordType.group,
"locations": CalRecordType.location,
"resources": CalRecordType.resource,
"addresses": CalRecordType.address,
}.get(recordTypeName, None)
if recordType is None:
log.error("Invalid Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
if recordType in types:
log.error("Duplicate Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
types.append(recordType)
directory.recordType = ConstantsContainer(types)
directory.fieldName = ConstantsContainer(
(directory.fieldName, CalFieldName))
fieldNames.append(directory.fieldName)
if cachingSeconds:
directory = CachingDirectoryService(
directory,
expireSeconds=cachingSeconds,
lookupsBetweenPurges=lookupsBetweenPurges,
negativeCaching=negativeCaching,
)
cachingServices.append(directory)
aggregatedServices.append(directory)
#
# Setup the Augment Service
#
serviceClass = {
"xml": "twistedcaldav.directory.augment.AugmentXMLDB",
}
for augmentFile in augmentServiceInfo.params.xmlFiles:
augmentFile = fullServerPath(dataRoot, augmentFile)
augmentFilePath = FilePath(augmentFile)
if not augmentFilePath.exists():
augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT)
augmentClass = namedClass(serviceClass[augmentServiceInfo.type])
log.info("Configuring augment service of type: {augmentClass}",
augmentClass=augmentClass)
try:
augmentService = augmentClass(**augmentServiceInfo.params)
except IOError:
log.error("Could not start augment service")
raise
userDirectory = None
for directory in aggregatedServices:
if RecordType.user in directory.recordTypes():
userDirectory = directory
break
else:
log.error("No directory service set up for users")
raise DirectoryConfigurationError
# Delegate service
delegateDirectory = DelegateDirectoryService(userDirectory.realmName,
store)
# (put at front of list so we don't try to ask the actual DS services
# about the delegate-related principals, for performance)
aggregatedServices.insert(0, delegateDirectory)
# Wiki service
if wikiServiceInfo.Enabled:
aggregatedServices.append(
WikiDirectoryService(
userDirectory.realmName,
wikiServiceInfo.EndpointDescriptor,
))
# Aggregate service
aggregateDirectory = AggregateDirectoryService(userDirectory.realmName,
aggregatedServices)
# Augment service
try:
fieldNames.append(CalFieldName)
augmented = AugmentedDirectoryService(aggregateDirectory, store,
augmentService)
augmented.fieldName = ConstantsContainer(fieldNames)
# The delegate directory needs a way to look up user/group records
# so hand it a reference to the augmented directory.
# FIXME: is there a better pattern to use here?
delegateDirectory.setMasterDirectory(augmented)
# Tell each caching service what method to use when reporting
# times and cache stats
for cachingService in cachingServices:
cachingService.setTimingMethod(augmented._addTiming)
# LDAP has additional stats to report
augmented._ldapDS = ldapService
except Exception as e:
log.error("Could not create directory service", error=e)
raise
if serversDB is not None:
augmented.setServersDB(serversDB)
if filterStartsWith:
augmented.setFilter(startswithFilter)
return augmented
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add a group
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(
groupID, _ignore_name, membershipHash, _ignore_modified,
extant
) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
self.assertTrue(extant)
# Remove the group
yield self.directory.removeRecords([u"testgroup"])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(
groupID, _ignore_name, membershipHash, _ignore_modified,
extant
) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = False
self.assertFalse(extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords(
(
TestRecord(
self.directory,
{
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}
),
),
create=True
)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(
groupID, _ignore_name, membershipHash, _ignore_modified,
extant
) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = True
self.assertTrue(extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(len(members), 100)
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add two groups
for uid in (u"testgroup", u"emptygroup",):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
# add members to test group
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def doWork(self):
self.transaction._groupCacher = groupCacher
return unpatchedDoWork(self)
groupCacher = self.groupCacher
unpatchedDoWork = GroupRefreshWork.doWork
self.patch(GroupRefreshWork, "doWork", doWork)
config.AutomaticPurging.Enabled = True
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid)
yield txn.commit()
self.assertTrue(group.extant)
# Remove the group
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = False
self.assertFalse(group.extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords(
(
TestRecord(
self.directory,
{
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}
),
),
create=True
)
if uid == u"testgroup":
group = yield self.directory.recordWithUID(uid)
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = True
self.assertTrue(group.extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(len(members), 100 if uid == u"testgroup" else 0)
@inlineCallbacks
def test_update_delete_unused(self):
"""
Verify that unused groups are deleted from group cache
"""
store = self.storeUnderTest()
# unused group deleted
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
# delegate groups not deleted
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
# delegate group is deleted. unused group is deleted
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
yield txn.removeDelegateGroup(delegator=u"sagen", delegateGroupID=testGroup.groupID, readWrite=True)
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertNotEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
@inlineCallbacks
def test_update_delete_old_nonextant(self):
"""
Verify that old missing groups are deleted from group cache
"""
oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds
store = self.storeUnderTest()
for uid in (u"testgroup", u"emptygroup",):
config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertTrue(group.extant)
# Remove the group, still cached
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertFalse(group.extant)
# delete the group
config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0"
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)