def test_cacher_off(self): """ Test that properties can still be read and written when the cacher is disabled. """ self.patch(self.store, "queryCacher", None) # Existing store - add a normal property self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10) self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) pname1 = propertyName("dummy1") pvalue1 = propertyValue("*") yield store1_user1.__setitem__(pname1, pvalue1) self.assertEqual(store1_user1[pname1], pvalue1) self.assertEqual(len(store1_user1._cached), 1) self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) yield self._txn.commit() self._txn = self.store.newTransaction() # Existing store - check a normal property self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10) self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) self.assertEqual(store1_user1[pname1], pvalue1)
def test_cacher_off(self): """ Test that properties can still be read and written when the cacher is disabled. """ self.patch(self.store, "queryCacher", None) # Existing store - add a normal property self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10) self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) pname1 = propertyName("dummy1") pvalue1 = propertyValue("*") yield store1_user1.__setitem__(pname1, pvalue1) self.assertEqual(store1_user1[pname1], pvalue1) self.assertEqual(len(store1_user1._cached), 1) self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) yield self._txn.commit() self._txn = self.store.newTransaction() # Existing store - check a normal property self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10) self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) self.assertEqual(store1_user1[pname1], pvalue1)
def _changed(self, store): if hasattr(self, "_txn"): yield self._txn.commit() delattr(self, "_txn") self._txn = self.store.newTransaction() store = self.propertyStore1 self.propertyStore = \ self.propertyStore1 = yield PropertyStore.load("user01", None, None, self._txn, 1) self.propertyStore1._shadowableKeys = store._shadowableKeys self.propertyStore1._proxyOverrideKeys = store._proxyOverrideKeys self.propertyStore1._globalKeys = store._globalKeys store = self.propertyStore2 self.propertyStore2 = yield PropertyStore.load("user01", "user02", None, self._txn, 1) self.propertyStore2._shadowableKeys = store._shadowableKeys self.propertyStore2._proxyOverrideKeys = store._proxyOverrideKeys self.propertyStore2._globalKeys = store._globalKeys store = self.propertyStore3 self.propertyStore3 = yield PropertyStore.load("user01", None, "user03", self._txn, 1) self.propertyStore3._shadowableKeys = store._shadowableKeys self.propertyStore3._proxyOverrideKeys = store._proxyOverrideKeys self.propertyStore3._globalKeys = store._globalKeys store = self.propertyStore4 self.propertyStore4 = yield PropertyStore.load("user01", "user02", "user04", self._txn, 1) self.propertyStore4._shadowableKeys = store._shadowableKeys self.propertyStore4._proxyOverrideKeys = store._proxyOverrideKeys self.propertyStore4._globalKeys = store._globalKeys
def setUp(self): self.notifierFactory = StubNotifierFactory() self.store = yield buildStore(self, self.notifierFactory) self.addCleanup(self.maybeCommitLast) self._txn = self.store.newTransaction() self.propertyStore = self.propertyStore1 = yield PropertyStore.load( "user01", None, self._txn, 1 ) self.propertyStore2 = yield PropertyStore.load("user01", "user02", self._txn, 1)
def setUp(self): self.notifierFactory = StubNotifierFactory() self.store = yield buildStore(self, self.notifierFactory) self.addCleanup(self.maybeCommitLast) self._txn = self.store.newTransaction() self.propertyStore = \ self.propertyStore1 = yield PropertyStore.load("user01", None, None, self._txn, 1) self.propertyStore2 = yield PropertyStore.load("user01", "user02", None, self._txn, 1) self.propertyStore3 = yield PropertyStore.load("user01", None, "user03", self._txn, 1) self.propertyStore4 = yield PropertyStore.load("user01", "user02", "user04", self._txn, 1)
def test_cacher_failure(self): """ Test that properties can still be read and written even when they are too larger for the cacher to handle. """ # Existing store - add a normal property self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10) self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) pname1 = propertyName("dummy1") pvalue1 = propertyValue("*") yield store1_user1.__setitem__(pname1, pvalue1) self.assertEqual(store1_user1[pname1], pvalue1) self.assertEqual(len(store1_user1._cached), 1) yield self._txn.commit() # Existing store - add a large property self._txn = self.store.newTransaction() self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10) self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) pname2 = propertyName("dummy2") pvalue2 = propertyValue("*" * (Memcacher.MEMCACHE_VALUE_LIMIT + 10)) yield store1_user1.__setitem__(pname2, pvalue2) self.assertEqual(store1_user1[pname2], pvalue2) self.assertEqual(len(store1_user1._cached), 2) yield self._txn.commit() # Try again - the cacher will fail large values self._txn = self.store.newTransaction() self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, self._txn, 10) self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache) self.assertEqual(store1_user1[pname1], pvalue1) self.assertEqual(store1_user1[pname2], pvalue2) self.assertEqual(len(store1_user1._cached), 2) yield store1_user1.__delitem__(pname1) self.assertTrue(pname1 not in store1_user1) yield store1_user1.__delitem__(pname2) self.assertTrue(pname2 not in store1_user1) self.assertEqual(len(store1_user1._cached), 0) self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
def test_cacher_failure(self): """ Test that properties can still be read and written even when they are too larger for the cacher to handle. """ # Existing store - add a normal property self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10) self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) pname1 = propertyName("dummy1") pvalue1 = propertyValue("*") yield store1_user1.__setitem__(pname1, pvalue1) self.assertEqual(store1_user1[pname1], pvalue1) self.assertEqual(len(store1_user1._cached), 1) yield self._txn.commit() # Existing store - add a large property self._txn = self.store.newTransaction() self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10) self.assertTrue("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) pname2 = propertyName("dummy2") pvalue2 = propertyValue("*" * (Memcacher.MEMCACHE_VALUE_LIMIT + 10)) yield store1_user1.__setitem__(pname2, pvalue2) self.assertEqual(store1_user1[pname2], pvalue2) self.assertEqual(len(store1_user1._cached), 2) yield self._txn.commit() # Try again - the cacher will fail large values self._txn = self.store.newTransaction() self.assertFalse("SQL.props:10/user01" in PropertyStore._cacher._memcacheProtocol._cache) store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 10) self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache) self.assertEqual(store1_user1[pname1], pvalue1) self.assertEqual(store1_user1[pname2], pvalue2) self.assertEqual(len(store1_user1._cached), 2) yield store1_user1.__delitem__(pname1) self.assertTrue(pname1 not in store1_user1) yield store1_user1.__delitem__(pname2) self.assertTrue(pname2 not in store1_user1) self.assertEqual(len(store1_user1._cached), 0) self.assertFalse("SQL.props:10/user01" in store1_user1._cacher._memcacheProtocol._cache)
def _loadPropertyStore(self): self._propertyStore = yield PropertyStore.load( self._ownerUID, self._ownerUID, None, self._txn, self._resourceID, notifyCallback=self.notifyChanged)
def _abort(self, store): if hasattr(self, "_txn"): yield self._txn.abort() delattr(self, "_txn") self._txn = self.store.newTransaction() store = self.propertyStore1 self.propertyStore = self.propertyStore1 = yield PropertyStore.load( "user01", None, self._txn, 1 ) self.propertyStore1._shadowableKeys = store._shadowableKeys self.propertyStore1._globalKeys = store._globalKeys store = self.propertyStore2 self.propertyStore2 = yield PropertyStore.load("user01", "user02", self._txn, 1) self.propertyStore2._shadowableKeys = store._shadowableKeys self.propertyStore2._globalKeys = store._globalKeys
def _loadPropertyStore(self): self._propertyStore = yield PropertyStore.load( self._ownerUID, self._ownerUID, None, self._txn, self._resourceID, notifyCallback=self.notifyChanged )
def _loadPropertyStore(self): """ No property store - stub to a NonePropertyStore. """ props = yield PropertyStore.load(self.uid(), self.uid(), None, self._txn, self._resourceID, notifyCallback=self.notifyChanged) self._propertyStore = props
def loadAllObjects(cls, parent): """ Load all child objects and return a list of them. This must create the child classes and initialize them using "batched" SQL operations to keep this constant wrt the number of children. This is an optimization for Depth:1 operations on the collection. """ results = [] # Load from the main table first dataRows = (yield cls._allColumnsByHomeIDQuery.on(parent._txn, homeID=parent._resourceID)) if dataRows: # Get property stores for all these child resources (if any found) propertyStores = (yield PropertyStore.forMultipleResources( parent.uid(), None, None, parent._txn, schema.NOTIFICATION.RESOURCE_ID, schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID, parent._resourceID, )) # Create the actual objects merging in properties for row in dataRows: child = cls(parent, None) ( child._resourceID, child._uid, child._md5, child._size, child._notificationType, child._created, child._modified, ) = tuple(row) child._created = parseSQLTimestamp(child._created) child._modified = parseSQLTimestamp(child._modified) try: child._notificationType = json.loads(child._notificationType) except ValueError: pass if isinstance(child._notificationType, unicode): child._notificationType = child._notificationType.encode( "utf-8") child._loadPropertyStore( props=propertyStores.get(child._resourceID, None)) results.append(child) returnValue(results)
def _loadPropertyStore(self): """ No property store - stub to a NonePropertyStore. """ props = yield PropertyStore.load( self.uid(), self.uid(), self._txn, self._resourceID, notifyCallback=self.notifyChanged ) self._propertyStore = props
def loadAllObjects(cls, parent): """ Load all child objects and return a list of them. This must create the child classes and initialize them using "batched" SQL operations to keep this constant wrt the number of children. This is an optimization for Depth:1 operations on the collection. """ results = [] # Load from the main table first dataRows = ( yield cls._allColumnsByHomeIDQuery.on(parent._txn, homeID=parent._resourceID)) if dataRows: # Get property stores for all these child resources (if any found) propertyStores = (yield PropertyStore.forMultipleResources( parent.uid(), None, None, parent._txn, schema.NOTIFICATION.RESOURCE_ID, schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID, parent._resourceID, )) # Create the actual objects merging in properties for row in dataRows: child = cls(parent, None) (child._resourceID, child._uid, child._md5, child._size, child._notificationType, child._created, child._modified,) = tuple(row) child._created = parseSQLTimestamp(child._created) child._modified = parseSQLTimestamp(child._modified) try: child._notificationType = json.loads(child._notificationType) except ValueError: pass if isinstance(child._notificationType, unicode): child._notificationType = child._notificationType.encode("utf-8") child._loadPropertyStore( props=propertyStores.get(child._resourceID, None) ) results.append(child) returnValue(results)
def test_insert_delete(self): # Existing store store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 2) pname = propertyName("dummy1") pvalue = propertyValue("value1-user1") yield store1_user1.__setitem__(pname, pvalue) self.assertEqual(store1_user1[pname], pvalue) yield store1_user1.__delitem__(pname) self.assertTrue(pname not in store1_user1) yield store1_user1.__setitem__(pname, pvalue) self.assertEqual(store1_user1[pname], pvalue)
def test_insert_delete(self): # Existing store store1_user1 = yield PropertyStore.load("user01", None, self._txn, 2) pname = propertyName("dummy1") pvalue = propertyValue("value1-user1") yield store1_user1.__setitem__(pname, pvalue) self.assertEqual(store1_user1[pname], pvalue) yield store1_user1.__delitem__(pname) self.assertTrue(pname not in store1_user1) yield store1_user1.__setitem__(pname, pvalue) self.assertEqual(store1_user1[pname], pvalue)
def test_concurrentInsertion(self): """ When two property stores set the same value, both should succeed, and update the cache. Whoever wins the race (i.e. updates last) will set the last property value. """ pname = propertyName("concurrent") pval1 = propertyValue("alpha") pval2 = propertyValue("beta") concurrentTxn = self.store.newTransaction() @inlineCallbacks def maybeAbortIt(): try: yield concurrentTxn.abort() except AlreadyFinishedError: pass self.addCleanup(maybeAbortIt) concurrentPropertyStore = yield PropertyStore.load( "user01", None, None, concurrentTxn, 1) concurrentPropertyStore[pname] = pval1 race = [] def tiebreaker(label): # Let's not get into the business of figuring out who the database # concurrency rules are supposed to pick; it might differ. We just # take the answer we're given for who gets to be the final writer, # and make sure that matches the property read in the next # transaction. def breaktie(result): race.append(label) return result return breaktie a = concurrentTxn.commit().addCallback(tiebreaker('a')) self.propertyStore[pname] = pval2 b = self._txn.commit().addCallback(tiebreaker('b')) del self._txn self.assertEquals((yield gatherResults([a, b])), [None, None]) yield self._abort(self.propertyStore) winner = {'a': pval1, 'b': pval2}[race[-1]] self.assertEquals(self.propertyStore[pname], winner)
def test_concurrentInsertion(self): """ When two property stores set the same value, both should succeed, and update the cache. Whoever wins the race (i.e. updates last) will set the last property value. """ pname = propertyName("concurrent") pval1 = propertyValue("alpha") pval2 = propertyValue("beta") concurrentTxn = self.store.newTransaction() @inlineCallbacks def maybeAbortIt(): try: yield concurrentTxn.abort() except AlreadyFinishedError: pass self.addCleanup(maybeAbortIt) concurrentPropertyStore = yield PropertyStore.load( "user01", None, concurrentTxn, 1 ) concurrentPropertyStore[pname] = pval1 race = [] def tiebreaker(label): # Let's not get into the business of figuring out who the database # concurrency rules are supposed to pick; it might differ. We just # take the answer we're given for who gets to be the final writer, # and make sure that matches the property read in the next # transaction. def breaktie(result): race.append(label) return result return breaktie a = concurrentTxn.commit().addCallback(tiebreaker('a')) self.propertyStore[pname] = pval2 b = self._txn.commit().addCallback(tiebreaker('b')) del self._txn self.assertEquals((yield gatherResults([a, b])), [None, None]) yield self._abort(self.propertyStore) winner = {'a': pval1, 'b': pval2}[race[-1]] self.assertEquals(self.propertyStore[pname], winner)
def test_copy(self): # Existing store store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 2) store1_user2 = yield PropertyStore.load("user01", "user02", None, self._txn, 2) # Populate current store with data props_user1 = ( (propertyName("dummy1"), propertyValue("value1-user1")), (propertyName("dummy2"), propertyValue("value2-user1")), ) props_user2 = ( (propertyName("dummy1"), propertyValue("value1-user2")), (propertyName("dummy3"), propertyValue("value3-user2")), ) for name, value in props_user1: store1_user1[name] = value for name, value in props_user2: store1_user2[name] = value yield self._txn.commit() self._txn = self.store.newTransaction() # Existing store store1_user1 = yield PropertyStore.load("user01", None, None, self._txn, 2) # New store store2_user1 = yield PropertyStore.load("user01", None, None, self._txn, 3) # Do copy and check results yield store2_user1.copyAllProperties(store1_user1) self.assertEqual(store1_user1.keys(), store2_user1.keys()) store1_user2 = yield PropertyStore.load("user01", "user02", None, self._txn, 2) store2_user2 = yield PropertyStore.load("user01", "user02", None, self._txn, 3) self.assertEqual(store1_user2.keys(), store2_user2.keys())
def test_copy(self): # Existing store store1_user1 = yield PropertyStore.load("user01", None, self._txn, 2) store1_user2 = yield PropertyStore.load("user01", "user02", self._txn, 2) # Populate current store with data props_user1 = ( (propertyName("dummy1"), propertyValue("value1-user1")), (propertyName("dummy2"), propertyValue("value2-user1")), ) props_user2 = ( (propertyName("dummy1"), propertyValue("value1-user2")), (propertyName("dummy3"), propertyValue("value3-user2")), ) for name, value in props_user1: store1_user1[name] = value for name, value in props_user2: store1_user2[name] = value yield self._txn.commit() self._txn = self.store.newTransaction() # Existing store store1_user1 = yield PropertyStore.load("user01", None, self._txn, 2) # New store store2_user1 = yield PropertyStore.load("user01", None, self._txn, 3) # Do copy and check results yield store2_user1.copyAllProperties(store1_user1) self.assertEqual(store1_user1.keys(), store2_user1.keys()) store1_user2 = yield PropertyStore.load("user01", "user02", self._txn, 2) store2_user2 = yield PropertyStore.load("user01", "user02", self._txn, 3) self.assertEqual(store1_user2.keys(), store2_user2.keys())