def test_cacher_off(self):
"""
Test that properties can still be read and written when the cacher is disabled.
"""
self.patch(self.store, "queryCacher", None)
# Existing store - add a normal property
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
pname1 = propertyName("dummy1")
pvalue1 = propertyValue("*")
yield store1_user1.__setitem__(pname1, pvalue1)
self.assertEqual(store1_user1[pname1], pvalue1)
self.assertEqual(len(store1_user1._cached), 1)
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
yield self._txn.commit()
self._txn = self.store.newTransaction()
# Existing store - check a normal property
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
self.assertEqual(store1_user1[pname1], pvalue1)
def test_cacher_off(self):
"""
Test that properties can still be read and written when the cacher is disabled.
"""
self.patch(self.store, "queryCacher", None)
# Existing store - add a normal property
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10)
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
pname1 = propertyName("dummy1")
pvalue1 = propertyValue("*")
yield store1_user1.__setitem__(pname1, pvalue1)
self.assertEqual(store1_user1[pname1], pvalue1)
self.assertEqual(len(store1_user1._cached), 1)
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
yield self._txn.commit()
self._txn = self.store.newTransaction()
# Existing store - check a normal property
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10)
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
self.assertEqual(store1_user1[pname1], pvalue1)
def _changed(self, store):
if hasattr(self, "_txn"):
yield self._txn.commit()
delattr(self, "_txn")
self._txn = self.store.newTransaction()
store = self.propertyStore1
self.propertyStore = \
self.propertyStore1 = yield PropertyStore.load("user01", None, None, self._txn, 1)
self.propertyStore1._shadowableKeys = store._shadowableKeys
self.propertyStore1._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore1._globalKeys = store._globalKeys
store = self.propertyStore2
self.propertyStore2 = yield PropertyStore.load("user01", "user02", None, self._txn, 1)
self.propertyStore2._shadowableKeys = store._shadowableKeys
self.propertyStore2._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore2._globalKeys = store._globalKeys
store = self.propertyStore3
self.propertyStore3 = yield PropertyStore.load("user01", None, "user03", self._txn, 1)
self.propertyStore3._shadowableKeys = store._shadowableKeys
self.propertyStore3._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore3._globalKeys = store._globalKeys
store = self.propertyStore4
self.propertyStore4 = yield PropertyStore.load("user01", "user02", "user04", self._txn, 1)
self.propertyStore4._shadowableKeys = store._shadowableKeys
self.propertyStore4._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore4._globalKeys = store._globalKeys
def _changed(self, store):
if hasattr(self, "_txn"):
yield self._txn.commit()
delattr(self, "_txn")
self._txn = self.store.newTransaction()
store = self.propertyStore1
self.propertyStore = \
self.propertyStore1 = yield PropertyStore.load("user01", None, None, self._txn, 1)
self.propertyStore1._shadowableKeys = store._shadowableKeys
self.propertyStore1._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore1._globalKeys = store._globalKeys
store = self.propertyStore2
self.propertyStore2 = yield PropertyStore.load("user01", "user02",
None, self._txn, 1)
self.propertyStore2._shadowableKeys = store._shadowableKeys
self.propertyStore2._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore2._globalKeys = store._globalKeys
store = self.propertyStore3
self.propertyStore3 = yield PropertyStore.load("user01", None,
"user03", self._txn, 1)
self.propertyStore3._shadowableKeys = store._shadowableKeys
self.propertyStore3._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore3._globalKeys = store._globalKeys
store = self.propertyStore4
self.propertyStore4 = yield PropertyStore.load("user01", "user02",
"user04", self._txn, 1)
self.propertyStore4._shadowableKeys = store._shadowableKeys
self.propertyStore4._proxyOverrideKeys = store._proxyOverrideKeys
self.propertyStore4._globalKeys = store._globalKeys
def setUp(self):
self.notifierFactory = StubNotifierFactory()
self.store = yield buildStore(self, self.notifierFactory)
self.addCleanup(self.maybeCommitLast)
self._txn = self.store.newTransaction()
self.propertyStore = self.propertyStore1 = yield PropertyStore.load(
"user01", None, self._txn, 1
)
self.propertyStore2 = yield PropertyStore.load("user01", "user02", self._txn, 1)
def setUp(self):
self.notifierFactory = StubNotifierFactory()
self.store = yield buildStore(self, self.notifierFactory)
self.addCleanup(self.maybeCommitLast)
self._txn = self.store.newTransaction()
self.propertyStore = \
self.propertyStore1 = yield PropertyStore.load("user01", None, None, self._txn, 1)
self.propertyStore2 = yield PropertyStore.load("user01", "user02", None, self._txn, 1)
self.propertyStore3 = yield PropertyStore.load("user01", None, "user03", self._txn, 1)
self.propertyStore4 = yield PropertyStore.load("user01", "user02", "user04", self._txn, 1)
def test_cacher_failure(self):
"""
Test that properties can still be read and written even when they are too larger for the
cacher to handle.
"""
# Existing store - add a normal property
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
pname1 = propertyName("dummy1")
pvalue1 = propertyValue("*")
yield store1_user1.__setitem__(pname1, pvalue1)
self.assertEqual(store1_user1[pname1], pvalue1)
self.assertEqual(len(store1_user1._cached), 1)
yield self._txn.commit()
# Existing store - add a large property
self._txn = self.store.newTransaction()
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
pname2 = propertyName("dummy2")
pvalue2 = propertyValue("*" * (Memcacher.MEMCACHE_VALUE_LIMIT + 10))
yield store1_user1.__setitem__(pname2, pvalue2)
self.assertEqual(store1_user1[pname2], pvalue2)
self.assertEqual(len(store1_user1._cached), 2)
yield self._txn.commit()
# Try again - the cacher will fail large values
self._txn = self.store.newTransaction()
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10)
self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
self.assertEqual(store1_user1[pname1], pvalue1)
self.assertEqual(store1_user1[pname2], pvalue2)
self.assertEqual(len(store1_user1._cached), 2)
yield store1_user1.__delitem__(pname1)
self.assertTrue(pname1 not in store1_user1)
yield store1_user1.__delitem__(pname2)
self.assertTrue(pname2 not in store1_user1)
self.assertEqual(len(store1_user1._cached), 0)
self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
def test_cacher_failure(self):
"""
Test that properties can still be read and written even when they are too larger for the
cacher to handle.
"""
# Existing store - add a normal property
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10)
self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
pname1 = propertyName("dummy1")
pvalue1 = propertyValue("*")
yield store1_user1.__setitem__(pname1, pvalue1)
self.assertEqual(store1_user1[pname1], pvalue1)
self.assertEqual(len(store1_user1._cached), 1)
yield self._txn.commit()
# Existing store - add a large property
self._txn = self.store.newTransaction()
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10)
self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
pname2 = propertyName("dummy2")
pvalue2 = propertyValue("*" * (Memcacher.MEMCACHE_VALUE_LIMIT + 10))
yield store1_user1.__setitem__(pname2, pvalue2)
self.assertEqual(store1_user1[pname2], pvalue2)
self.assertEqual(len(store1_user1._cached), 2)
yield self._txn.commit()
# Try again - the cacher will fail large values
self._txn = self.store.newTransaction()
self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache)
store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10)
self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
self.assertEqual(store1_user1[pname1], pvalue1)
self.assertEqual(store1_user1[pname2], pvalue2)
self.assertEqual(len(store1_user1._cached), 2)
yield store1_user1.__delitem__(pname1)
self.assertTrue(pname1 not in store1_user1)
yield store1_user1.__delitem__(pname2)
self.assertTrue(pname2 not in store1_user1)
self.assertEqual(len(store1_user1._cached), 0)
self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
def _loadPropertyStore(self):
self._propertyStore = yield PropertyStore.load(
self._ownerUID,
self._ownerUID,
None,
self._txn,
self._resourceID,
notifyCallback=self.notifyChanged)
def _abort(self, store):
if hasattr(self, "_txn"):
yield self._txn.abort()
delattr(self, "_txn")
self._txn = self.store.newTransaction()
store = self.propertyStore1
self.propertyStore = self.propertyStore1 = yield PropertyStore.load(
"user01", None, self._txn, 1
)
self.propertyStore1._shadowableKeys = store._shadowableKeys
self.propertyStore1._globalKeys = store._globalKeys
store = self.propertyStore2
self.propertyStore2 = yield PropertyStore.load("user01", "user02", self._txn, 1)
self.propertyStore2._shadowableKeys = store._shadowableKeys
self.propertyStore2._globalKeys = store._globalKeys
def _loadPropertyStore(self):
self._propertyStore = yield PropertyStore.load(
self._ownerUID,
self._ownerUID,
None,
self._txn,
self._resourceID,
notifyCallback=self.notifyChanged
)
def _loadPropertyStore(self):
"""
No property store - stub to a NonePropertyStore.
"""
props = yield PropertyStore.load(self.uid(),
self.uid(),
None,
self._txn,
self._resourceID,
notifyCallback=self.notifyChanged)
self._propertyStore = props
def loadAllObjects(cls, parent):
"""
Load all child objects and return a list of them. This must create the
child classes and initialize them using "batched" SQL operations to keep
this constant wrt the number of children. This is an optimization for
Depth:1 operations on the collection.
"""
results = []
# Load from the main table first
dataRows = (yield
cls._allColumnsByHomeIDQuery.on(parent._txn,
homeID=parent._resourceID))
if dataRows:
# Get property stores for all these child resources (if any found)
propertyStores = (yield PropertyStore.forMultipleResources(
parent.uid(),
None,
None,
parent._txn,
schema.NOTIFICATION.RESOURCE_ID,
schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID,
parent._resourceID,
))
# Create the actual objects merging in properties
for row in dataRows:
child = cls(parent, None)
(
child._resourceID,
child._uid,
child._md5,
child._size,
child._notificationType,
child._created,
child._modified,
) = tuple(row)
child._created = parseSQLTimestamp(child._created)
child._modified = parseSQLTimestamp(child._modified)
try:
child._notificationType = json.loads(child._notificationType)
except ValueError:
pass
if isinstance(child._notificationType, unicode):
child._notificationType = child._notificationType.encode(
"utf-8")
child._loadPropertyStore(
props=propertyStores.get(child._resourceID, None))
results.append(child)
returnValue(results)
def _loadPropertyStore(self):
"""
No property store - stub to a NonePropertyStore.
"""
props = yield PropertyStore.load(
self.uid(),
self.uid(),
self._txn,
self._resourceID,
notifyCallback=self.notifyChanged
)
self._propertyStore = props
def loadAllObjects(cls, parent):
"""
Load all child objects and return a list of them. This must create the
child classes and initialize them using "batched" SQL operations to keep
this constant wrt the number of children. This is an optimization for
Depth:1 operations on the collection.
"""
results = []
# Load from the main table first
dataRows = (
yield cls._allColumnsByHomeIDQuery.on(parent._txn,
homeID=parent._resourceID))
if dataRows:
# Get property stores for all these child resources (if any found)
propertyStores = (yield PropertyStore.forMultipleResources(
parent.uid(),
None,
None,
parent._txn,
schema.NOTIFICATION.RESOURCE_ID,
schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID,
parent._resourceID,
))
# Create the actual objects merging in properties
for row in dataRows:
child = cls(parent, None)
(child._resourceID,
child._uid,
child._md5,
child._size,
child._notificationType,
child._created,
child._modified,) = tuple(row)
child._created = parseSQLTimestamp(child._created)
child._modified = parseSQLTimestamp(child._modified)
try:
child._notificationType = json.loads(child._notificationType)
except ValueError:
pass
if isinstance(child._notificationType, unicode):
child._notificationType = child._notificationType.encode("utf-8")
child._loadPropertyStore(
props=propertyStores.get(child._resourceID, None)
)
results.append(child)
returnValue(results)
def test_insert_delete(self):
# Existing store
store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 2)
pname = propertyName("dummy1")
pvalue = propertyValue("value1-user1")
yield store1_user1.__setitem__(pname, pvalue)
self.assertEqual(store1_user1[pname], pvalue)
yield store1_user1.__delitem__(pname)
self.assertTrue(pname not in store1_user1)
yield store1_user1.__setitem__(pname, pvalue)
self.assertEqual(store1_user1[pname], pvalue)
def test_insert_delete(self):
# Existing store
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 2)
pname = propertyName("dummy1")
pvalue = propertyValue("value1-user1")
yield store1_user1.__setitem__(pname, pvalue)
self.assertEqual(store1_user1[pname], pvalue)
yield store1_user1.__delitem__(pname)
self.assertTrue(pname not in store1_user1)
yield store1_user1.__setitem__(pname, pvalue)
self.assertEqual(store1_user1[pname], pvalue)
def test_concurrentInsertion(self):
"""
When two property stores set the same value, both should succeed, and
update the cache. Whoever wins the race (i.e. updates last) will set
the last property value.
"""
pname = propertyName("concurrent")
pval1 = propertyValue("alpha")
pval2 = propertyValue("beta")
concurrentTxn = self.store.newTransaction()
@inlineCallbacks
def maybeAbortIt():
try:
yield concurrentTxn.abort()
except AlreadyFinishedError:
pass
self.addCleanup(maybeAbortIt)
concurrentPropertyStore = yield PropertyStore.load(
"user01", None, None, concurrentTxn, 1)
concurrentPropertyStore[pname] = pval1
race = []
def tiebreaker(label):
# Let's not get into the business of figuring out who the database
# concurrency rules are supposed to pick; it might differ. We just
# take the answer we're given for who gets to be the final writer,
# and make sure that matches the property read in the next
# transaction.
def breaktie(result):
race.append(label)
return result
return breaktie
a = concurrentTxn.commit().addCallback(tiebreaker('a'))
self.propertyStore[pname] = pval2
b = self._txn.commit().addCallback(tiebreaker('b'))
del self._txn
self.assertEquals((yield gatherResults([a, b])), [None, None])
yield self._abort(self.propertyStore)
winner = {'a': pval1, 'b': pval2}[race[-1]]
self.assertEquals(self.propertyStore[pname], winner)
def test_concurrentInsertion(self):
"""
When two property stores set the same value, both should succeed, and
update the cache. Whoever wins the race (i.e. updates last) will set
the last property value.
"""
pname = propertyName("concurrent")
pval1 = propertyValue("alpha")
pval2 = propertyValue("beta")
concurrentTxn = self.store.newTransaction()
@inlineCallbacks
def maybeAbortIt():
try:
yield concurrentTxn.abort()
except AlreadyFinishedError:
pass
self.addCleanup(maybeAbortIt)
concurrentPropertyStore = yield PropertyStore.load(
"user01", None, concurrentTxn, 1
)
concurrentPropertyStore[pname] = pval1
race = []
def tiebreaker(label):
# Let's not get into the business of figuring out who the database
# concurrency rules are supposed to pick; it might differ. We just
# take the answer we're given for who gets to be the final writer,
# and make sure that matches the property read in the next
# transaction.
def breaktie(result):
race.append(label)
return result
return breaktie
a = concurrentTxn.commit().addCallback(tiebreaker('a'))
self.propertyStore[pname] = pval2
b = self._txn.commit().addCallback(tiebreaker('b'))
del self._txn
self.assertEquals((yield gatherResults([a, b])), [None, None])
yield self._abort(self.propertyStore)
winner = {'a': pval1,
'b': pval2}[race[-1]]
self.assertEquals(self.propertyStore[pname], winner)
def test_copy(self):
# Existing store
store1_user1 = yield PropertyStore.load("user01", None, None,
self._txn, 2)
store1_user2 = yield PropertyStore.load("user01", "user02", None,
self._txn, 2)
# Populate current store with data
props_user1 = (
(propertyName("dummy1"), propertyValue("value1-user1")),
(propertyName("dummy2"), propertyValue("value2-user1")),
)
props_user2 = (
(propertyName("dummy1"), propertyValue("value1-user2")),
(propertyName("dummy3"), propertyValue("value3-user2")),
)
for name, value in props_user1:
store1_user1[name] = value
for name, value in props_user2:
store1_user2[name] = value
yield self._txn.commit()
self._txn = self.store.newTransaction()
# Existing store
store1_user1 = yield PropertyStore.load("user01", None, None,
self._txn, 2)
# New store
store2_user1 = yield PropertyStore.load("user01", None, None,
self._txn, 3)
# Do copy and check results
yield store2_user1.copyAllProperties(store1_user1)
self.assertEqual(store1_user1.keys(), store2_user1.keys())
store1_user2 = yield PropertyStore.load("user01", "user02", None,
self._txn, 2)
store2_user2 = yield PropertyStore.load("user01", "user02", None,
self._txn, 3)
self.assertEqual(store1_user2.keys(), store2_user2.keys())
def test_copy(self):
# Existing store
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 2)
store1_user2 = yield PropertyStore.load("user01", "user02", self._txn, 2)
# Populate current store with data
props_user1 = (
(propertyName("dummy1"), propertyValue("value1-user1")),
(propertyName("dummy2"), propertyValue("value2-user1")),
)
props_user2 = (
(propertyName("dummy1"), propertyValue("value1-user2")),
(propertyName("dummy3"), propertyValue("value3-user2")),
)
for name, value in props_user1:
store1_user1[name] = value
for name, value in props_user2:
store1_user2[name] = value
yield self._txn.commit()
self._txn = self.store.newTransaction()
# Existing store
store1_user1 = yield PropertyStore.load("user01", None, self._txn, 2)
# New store
store2_user1 = yield PropertyStore.load("user01", None, self._txn, 3)
# Do copy and check results
yield store2_user1.copyAllProperties(store1_user1)
self.assertEqual(store1_user1.keys(), store2_user1.keys())
store1_user2 = yield PropertyStore.load("user01", "user02", self._txn, 2)
store2_user2 = yield PropertyStore.load("user01", "user02", self._txn, 3)
self.assertEqual(store1_user2.keys(), store2_user2.keys())
