def _doBatchRefresh(self):
"""
Do refresh of attendees in batches until the batch list is empty.
"""
# Need to lock whilst manipulating the batch list
log.debug("ImplicitProcessing - batch refresh for UID: '%s'" % (self.uid,))
lock = MemcacheLock(
"BatchRefreshUIDLock",
self.uid,
timeout=config.Scheduling.Options.UIDLockTimeoutSeconds,
expire_time=config.Scheduling.Options.UIDLockExpirySeconds,
)
try:
yield lock.acquire()
except MemcacheLockTimeoutError:
# If we could not lock then just fail the refresh - not sure what else to do
returnValue(None)
try:
# Get the batch list
cache = Memcacher("BatchRefreshAttendees", pickle=True)
pendingAttendees = yield cache.get(self.uid)
if pendingAttendees:
# Get the next batch of attendees to process and update the cache value or remove it if
# no more processing is needed
attendeesToProcess = pendingAttendees[:config.Scheduling.Options.AttendeeRefreshBatch]
pendingAttendees = pendingAttendees[config.Scheduling.Options.AttendeeRefreshBatch:]
if pendingAttendees:
yield cache.set(self.uid, pendingAttendees)
else:
yield cache.delete(self.uid)
# Make sure we release this here to avoid potential deadlock when grabbing the ImplicitUIDLock in the next call
yield lock.release()
# Now do the batch refresh
yield self._doDelayedRefresh(attendeesToProcess)
# Queue the next refresh if needed
if pendingAttendees:
self._enqueueBatchRefresh()
else:
yield cache.delete(self.uid)
yield lock.release()
finally:
yield lock.clean()
def test_all_pickled(self):
for processType in (
"Single",
"Combined",
):
config.ProcessType = processType
cacher = Memcacher("testing", pickle=True)
result = yield cacher.set("akey", [
"1",
"2",
"3",
])
self.assertTrue(result)
result = yield cacher.get("akey")
if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher):
self.assertEquals(None, result)
else:
self.assertEquals([
"1",
"2",
"3",
], result)
result = yield cacher.delete("akey")
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(None, result)
def test_all_noinvalidation(self):
for processType in (
"Single",
"Combined",
):
config.ProcessType = processType
cacher = Memcacher("testing", no_invalidation=True)
result = yield cacher.set("akey", [
"1",
"2",
"3",
])
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals([
"1",
"2",
"3",
], result)
result = yield cacher.delete("akey")
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(None, result)
def test_all_noinvalidation(self):
for processType in ("Single", "Combined",):
config.ProcessType = processType
cacher = Memcacher("testing", pickle=True, no_invalidation=True)
result = yield cacher.set("akey", ["1", "2", "3", ])
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(["1", "2", "3", ], result)
result = yield cacher.delete("akey")
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(None, result)
def test_all_pickled(self):
for processType in ("Single", "Combined",):
config.ProcessType = processType
cacher = Memcacher("testing", pickle=True)
result = yield cacher.set("akey", ["1", "2", "3", ])
self.assertTrue(result)
result = yield cacher.get("akey")
if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher):
self.assertEquals(None, result)
else:
self.assertEquals(["1", "2", "3", ], result)
result = yield cacher.delete("akey")
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(None, result)
def test_delete(self):
for processType in ("Single", "Combined",):
config.ProcessType = processType
cacher = Memcacher("testing")
result = yield cacher.set("akey", "avalue")
self.assertTrue(result)
result = yield cacher.get("akey")
if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher):
self.assertEquals(None, result)
else:
self.assertEquals("avalue", result)
result = yield cacher.delete("akey")
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(None, result)
def test_delete(self):
for processType in (
"Single",
"Combined",
):
config.ProcessType = processType
cacher = Memcacher("testing")
result = yield cacher.set("akey", "avalue")
self.assertTrue(result)
result = yield cacher.get("akey")
if isinstance(cacher._memcacheProtocol, Memcacher.nullCacher):
self.assertEquals(None, result)
else:
self.assertEquals("avalue", result)
result = yield cacher.delete("akey")
self.assertTrue(result)
result = yield cacher.get("akey")
self.assertEquals(None, result)