def setUp(self):
self.numUsers = 1000
# The "local" directory service
self.directory = DirectoryService(None)
# The "remote" directory service
remoteDirectory = CalendarInMemoryDirectoryService(None)
# Add users
records = []
fieldName = remoteDirectory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
remoteDirectory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add a big group
records.append(
TestRecord(
remoteDirectory, {
fieldName.uid: u"bigGroup",
fieldName.recordType: RecordType.group,
}))
yield remoteDirectory.updateRecords(records, create=True)
group = yield remoteDirectory.recordWithUID(u"bigGroup")
members = yield remoteDirectory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
# Connect the two services directly via an IOPump
client = AMP()
self.server = DirectoryProxyAMPProtocol(remoteDirectory)
pump = returnConnected(self.server, client)
# Replace the normal _getConnection method with one that bypasses any
# actual networking
self.patch(self.directory, "_getConnection", lambda: succeed(client))
# Wrap the normal _call method with one that flushes the IOPump
# afterwards
origCall = self.directory._call
def newCall(*args, **kwds):
d = origCall(*args, **kwds)
pump.flush()
return d
self.patch(self.directory, "_call", newCall)
def setUp(self):
self.numUsers = 1000
# The "local" directory service
self.directory = DirectoryService(None)
# The "remote" directory service
remoteDirectory = CalendarInMemoryDirectoryService(None)
# Add users
records = []
fieldName = remoteDirectory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
remoteDirectory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add a big group
records.append(
TestRecord(
remoteDirectory,
{
fieldName.uid: u"bigGroup",
fieldName.recordType: RecordType.group,
}
)
)
yield remoteDirectory.updateRecords(records, create=True)
group = yield remoteDirectory.recordWithUID(u"bigGroup")
members = yield remoteDirectory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
# Connect the two services directly via an IOPump
client = AMP()
self.server = DirectoryProxyAMPProtocol(remoteDirectory)
pump = returnConnected(self.server, client)
# Replace the normal _getConnection method with one that bypasses any
# actual networking
self.patch(self.directory, "_getConnection", lambda: succeed(client))
# Wrap the normal _call method with one that flushes the IOPump
# afterwards
origCall = self.directory._call
def newCall(*args, **kwds):
d = origCall(*args, **kwds)
pump.flush()
return d
self.patch(self.directory, "_call", newCall)
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add two groups
for uid in (
u"testgroup",
u"emptygroup",
):
records.append(
TestRecord(self.directory, {
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}))
yield self.directory.updateRecords(records, create=True)
# add members to test group
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def doWork(self):
self.transaction._groupCacher = groupCacher
return unpatchedDoWork(self)
groupCacher = self.groupCacher
unpatchedDoWork = GroupRefreshWork.doWork
self.patch(GroupRefreshWork, "doWork", doWork)
config.AutomaticPurging.Enabled = True
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
for uid in (
u"testgroup",
u"emptygroup",
):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid)
yield txn.commit()
self.assertTrue(group.extant)
# Remove the group
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = False
self.assertFalse(group.extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords(
(TestRecord(self.directory, {
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}), ),
create=True)
if uid == u"testgroup":
group = yield self.directory.recordWithUID(uid)
members = yield self.directory.recordsWithRecordType(
RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = True
self.assertTrue(group.extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(len(members), 100 if uid == u"testgroup" else 0)
@inlineCallbacks
def test_update_delete_unused(self):
"""
Verify that unused groups are deleted from group cache
"""
store = self.storeUnderTest()
# unused group deleted
for uid in (
u"testgroup",
u"emptygroup",
):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
# delegate groups not deleted
for uid in (
u"testgroup",
u"emptygroup",
):
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen",
delegateGroupID=group.groupID,
readWrite=True)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
# delegate group is deleted. unused group is deleted
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
yield txn.removeDelegateGroup(delegator=u"sagen",
delegateGroupID=testGroup.groupID,
readWrite=True)
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertNotEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
@inlineCallbacks
def test_update_delete_old_nonextant(self):
"""
Verify that old missing groups are deleted from group cache
"""
oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds
store = self.storeUnderTest()
for uid in (
u"testgroup",
u"emptygroup",
):
config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen",
delegateGroupID=group.groupID,
readWrite=True)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertTrue(group.extant)
# Remove the group, still cached
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertFalse(group.extant)
# delete the group
config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0"
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames:
(u"foo{ctr:05d}".format(ctr=i), ),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i), ),
fieldName.recordType: RecordType.user,
}))
# Add a group
records.append(
TestRecord(
self.directory, {
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}))
yield self.directory.updateRecords(records, create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(groupID, _ignore_name, membershipHash, _ignore_modified,
extant) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
self.assertTrue(extant)
# Remove the group
yield self.directory.removeRecords([u"testgroup"])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(groupID, _ignore_name, membershipHash, _ignore_modified,
extant) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = False
self.assertFalse(extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords((TestRecord(
self.directory, {
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}), ),
create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(groupID, _ignore_name, membershipHash, _ignore_modified,
extant) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = True
self.assertTrue(extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(len(members), 100)
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add a group
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(
groupID, _ignore_name, membershipHash, _ignore_modified,
extant
) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
self.assertTrue(extant)
# Remove the group
yield self.directory.removeRecords([u"testgroup"])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(
groupID, _ignore_name, membershipHash, _ignore_modified,
extant
) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = False
self.assertFalse(extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords(
(
TestRecord(
self.directory,
{
fieldName.uid: u"testgroup",
fieldName.recordType: RecordType.group,
}
),
),
create=True
)
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, u"testgroup")
(
groupID, _ignore_name, membershipHash, _ignore_modified,
extant
) = (yield txn.groupByUID(u"testgroup"))
yield txn.commit()
# Extant = True
self.assertTrue(extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.membersOfGroup(groupID)
yield txn.commit()
self.assertEquals(len(members), 100)
class DynamicGroupTest(StoreTestCase):
@inlineCallbacks
def setUp(self):
yield super(DynamicGroupTest, self).setUp()
self.directory = CalendarInMemoryDirectoryService(None)
self.store.setDirectoryService(self.directory)
self.groupCacher = GroupCacher(self.directory)
self.numUsers = 100
# Add users
records = []
fieldName = self.directory.fieldName
for i in xrange(self.numUsers):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: u"foo{ctr:05d}".format(ctr=i),
fieldName.shortNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.fullNames: (u"foo{ctr:05d}".format(ctr=i),),
fieldName.recordType: RecordType.user,
}
)
)
# Add two groups
for uid in (u"testgroup", u"emptygroup",):
records.append(
TestRecord(
self.directory,
{
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}
)
)
yield self.directory.updateRecords(records, create=True)
# add members to test group
group = yield self.directory.recordWithUID(u"testgroup")
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
def doWork(self):
self.transaction._groupCacher = groupCacher
return unpatchedDoWork(self)
groupCacher = self.groupCacher
unpatchedDoWork = GroupRefreshWork.doWork
self.patch(GroupRefreshWork, "doWork", doWork)
config.AutomaticPurging.Enabled = True
@inlineCallbacks
def test_extant(self):
"""
Verify that once a group is removed from the directory, the next call
to refreshGroup() will set the "extent" to False. Add the group back
to the directory and "extent" becomes True.
"""
store = self.storeUnderTest()
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid)
yield txn.commit()
self.assertTrue(group.extant)
# Remove the group
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = False
self.assertFalse(group.extant)
# The list of members stored in the DB for this group is now empty
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(members, set())
# Add the group back into the directory
fieldName = self.directory.fieldName
yield self.directory.updateRecords(
(
TestRecord(
self.directory,
{
fieldName.uid: uid,
fieldName.recordType: RecordType.group,
}
),
),
create=True
)
if uid == u"testgroup":
group = yield self.directory.recordWithUID(uid)
members = yield self.directory.recordsWithRecordType(RecordType.user)
yield group.setMembers(members)
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = (yield txn.groupByUID(uid))
yield txn.commit()
# Extant = True
self.assertTrue(group.extant)
# The list of members stored in the DB for this group has 100 users
txn = store.newTransaction()
members = yield txn.groupMemberUIDs(group.groupID)
yield txn.commit()
self.assertEquals(len(members), 100 if uid == u"testgroup" else 0)
@inlineCallbacks
def test_update_delete_unused(self):
"""
Verify that unused groups are deleted from group cache
"""
store = self.storeUnderTest()
# unused group deleted
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
yield self.groupCacher.refreshGroup(txn, uid)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
# delegate groups not deleted
for uid in (u"testgroup", u"emptygroup",):
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True)
yield txn.commit()
self.assertNotEqual(group, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
# delegate group is deleted. unused group is deleted
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
yield txn.removeDelegateGroup(delegator=u"sagen", delegateGroupID=testGroup.groupID, readWrite=True)
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertNotEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
txn = store.newTransaction()
yield self.groupCacher.update(txn)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
testGroup = yield txn.groupByUID(u"testgroup", create=False)
emptyGroup = yield txn.groupByUID(u"emptygroup", create=False)
yield txn.commit()
self.assertEqual(testGroup, None)
self.assertNotEqual(emptyGroup, None)
@inlineCallbacks
def test_update_delete_old_nonextant(self):
"""
Verify that old missing groups are deleted from group cache
"""
oldGroupPurgeIntervalSeconds = config.AutomaticPurging.GroupPurgeIntervalSeconds
store = self.storeUnderTest()
for uid in (u"testgroup", u"emptygroup",):
config.AutomaticPurging.GroupPurgeIntervalSeconds = oldGroupPurgeIntervalSeconds
txn = store.newTransaction()
group = yield txn.groupByUID(uid)
yield txn.addDelegateGroup(delegator=u"sagen", delegateGroupID=group.groupID, readWrite=True)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertTrue(group.extant)
# Remove the group, still cached
yield self.directory.removeRecords([uid])
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
yield JobItem.waitEmpty(store.newTransaction, reactor, 60)
txn = store.newTransaction()
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertNotEqual(group, None)
self.assertFalse(group.extant)
# delete the group
config.AutomaticPurging.GroupPurgeIntervalSeconds = "0.0"
txn = store.newTransaction()
yield self.groupCacher.update(txn)
group = yield txn.groupByUID(uid, create=False)
yield txn.commit()
self.assertEqual(group, None)
