def _doBatchRefresh(self): """ Do refresh of attendees in batches until the batch list is empty. """ # Need to lock whilst manipulating the batch list log.debug("ImplicitProcessing - batch refresh for UID: '%s'" % (self.uid,)) lock = MemcacheLock( "BatchRefreshUIDLock", self.uid, timeout=config.Scheduling.Options.UIDLockTimeoutSeconds, expire_time=config.Scheduling.Options.UIDLockExpirySeconds, ) try: yield lock.acquire() except MemcacheLockTimeoutError: # If we could not lock then just fail the refresh - not sure what else to do returnValue(None) try: # Get the batch list cache = Memcacher("BatchRefreshAttendees", pickle=True) pendingAttendees = yield cache.get(self.uid) if pendingAttendees: # Get the next batch of attendees to process and update the cache value or remove it if # no more processing is needed attendeesToProcess = pendingAttendees[:config.Scheduling.Options.AttendeeRefreshBatch] pendingAttendees = pendingAttendees[config.Scheduling.Options.AttendeeRefreshBatch:] if pendingAttendees: yield cache.set(self.uid, pendingAttendees) else: yield cache.delete(self.uid) # Make sure we release this here to avoid potential deadlock when grabbing the ImplicitUIDLock in the next call yield lock.release() # Now do the batch refresh yield self._doDelayedRefresh(attendeesToProcess) # Queue the next refresh if needed if pendingAttendees: self._enqueueBatchRefresh() else: yield cache.delete(self.uid) yield lock.release() finally: yield lock.clean()
def test_all_pickled(self): for processType in ( "Single", "Combined", ): config.ProcessType = processType cacher = Memcacher("testing", pickle=True) result = yield cacher.set("akey", [ "1", "2", "3", ]) self.assertTrue(result) result = yield cacher.get("akey") if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher): self.assertEquals(None, result) else: self.assertEquals([ "1", "2", "3", ], result) result = yield cacher.delete("akey") self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(None, result)
def test_all_noinvalidation(self): for processType in ( "Single", "Combined", ): config.ProcessType = processType cacher = Memcacher("testing", no_invalidation=True) result = yield cacher.set("akey", [ "1", "2", "3", ]) self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals([ "1", "2", "3", ], result) result = yield cacher.delete("akey") self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(None, result)
def test_all_noinvalidation(self): for processType in ("Single", "Combined",): config.ProcessType = processType cacher = Memcacher("testing", pickle=True, no_invalidation=True) result = yield cacher.set("akey", ["1", "2", "3", ]) self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(["1", "2", "3", ], result) result = yield cacher.delete("akey") self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(None, result)
def test_all_pickled(self): for processType in ("Single", "Combined",): config.ProcessType = processType cacher = Memcacher("testing", pickle=True) result = yield cacher.set("akey", ["1", "2", "3", ]) self.assertTrue(result) result = yield cacher.get("akey") if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher): self.assertEquals(None, result) else: self.assertEquals(["1", "2", "3", ], result) result = yield cacher.delete("akey") self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(None, result)
def test_delete(self): for processType in ("Single", "Combined",): config.ProcessType = processType cacher = Memcacher("testing") result = yield cacher.set("akey", "avalue") self.assertTrue(result) result = yield cacher.get("akey") if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher): self.assertEquals(None, result) else: self.assertEquals("avalue", result) result = yield cacher.delete("akey") self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(None, result)
def test_delete(self): for processType in ( "Single", "Combined", ): config.ProcessType = processType cacher = Memcacher("testing") result = yield cacher.set("akey", "avalue") self.assertTrue(result) result = yield cacher.get("akey") if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher): self.assertEquals(None, result) else: self.assertEquals("avalue", result) result = yield cacher.delete("akey") self.assertTrue(result) result = yield cacher.get("akey") self.assertEquals(None, result)