def upgradeProcessor1to2(oldProcessor):
"""
Batch processors stopped polling at version 2, so they no longer needed the
idleInterval attribute. They also gained a scheduled attribute which
tracks their interaction with the scheduler. Since they stopped polling,
we also set them up as a timed event here to make sure that they don't
silently disappear, never to be seen again: running them with the scheduler
gives them a chance to figure out what's up and set up whatever other state
they need to continue to run.
Since this introduces a new dependency of all batch processors on a powerup
for the IScheduler, install a Scheduler or a SubScheduler if one is not
already present.
"""
newProcessor = oldProcessor.upgradeVersion(
oldProcessor.typeName, 1, 2,
busyInterval=oldProcessor.busyInterval)
newProcessor.scheduled = extime.Time()
s = newProcessor.store
sch = iaxiom.IScheduler(s, None)
if sch is None:
if s.parent is None:
# Only site stores have no parents.
sch = Scheduler(store=s)
else:
# Substores get subschedulers.
sch = SubScheduler(store=s)
installOn(sch, s)
# And set it up to run.
sch.schedule(newProcessor, newProcessor.scheduled)
return newProcessor
def upgradeProcessor1to2(oldProcessor):
"""
Batch processors stopped polling at version 2, so they no longer needed the
idleInterval attribute. They also gained a scheduled attribute which
tracks their interaction with the scheduler. Since they stopped polling,
we also set them up as a timed event here to make sure that they don't
silently disappear, never to be seen again: running them with the scheduler
gives them a chance to figure out what's up and set up whatever other state
they need to continue to run.
Since this introduces a new dependency of all batch processors on a powerup
for the IScheduler, install a Scheduler or a SubScheduler if one is not
already present.
"""
newProcessor = oldProcessor.upgradeVersion(
oldProcessor.typeName, 1, 2,
busyInterval=oldProcessor.busyInterval)
newProcessor.scheduled = extime.Time()
s = newProcessor.store
sch = iaxiom.IScheduler(s, None)
if sch is None:
if s.parent is None:
# Only site stores have no parents.
sch = Scheduler(store=s)
else:
# Substores get subschedulers.
sch = SubScheduler(store=s)
installOn(sch, s)
# And set it up to run.
sch.schedule(newProcessor, newProcessor.scheduled)
return newProcessor
def makeScheduler(self, install=True):
"""
Create, install, and return a Scheduler with a fake callLater.
"""
scheduler = Scheduler(store=self.store)
scheduler.callLater = self.clock.callLater
scheduler.now = self.now
if install:
installOn(scheduler, self.store)
return scheduler
def setUp(self):
self.clock = Clock()
scheduler = Scheduler(store=self.siteStore)
self.stubTime(scheduler)
installOn(scheduler, self.siteStore)
IService(self.siteStore).startService()
def createDatabase(s):
"""
Create a store containing a MailingListFilteringPowerup and its
dependencies.
"""
Scheduler(store=s).installOn(s)
MessageSource(store=s)
tc = Catalog(store=s)
rfp = MailingListFilteringPowerup(store=s,
tagCatalog=tc).installOn(s)
def setUp(self):
"""
Create a store with a scheduler installed on it and hook the C{now} and
C{callLater} methods of that scheduler so their behavior can be
controlled by these tests.
"""
self.calls = []
self.store = Store(filepath.FilePath(self.mktemp()))
self.siteScheduler = Scheduler(store=self.store)
installOn(self.siteScheduler, self.store)
self.siteScheduler.callLater = self._callLater
def setUp(self):
dbdir = self.mktemp()
self.store = s = store.Store(dbdir)
installOn(Scheduler(store=s), s)
ls = userbase.LoginSystem(store=s)
installOn(ls, s)
acc = ls.addAccount('username', 'dom.ain', 'password')
ss = acc.avatars.open()
self.df = spam.DSPAMFilter(store=ss)
installOn(self.df, ss)
self.f = self.df.filter
def setUp(self):
self.dbdir = FilePath(self.mktemp())
self.store = Store(self.dbdir)
self.ls = userbase.LoginSystem(store=self.store)
self.scheduler = Scheduler(store=self.store)
dependency.installOn(self.scheduler, self.store)
self.account = self.ls.addAccount(
self.localpart, self.domain, u'PASSWORD')
self.accountStore = self.account.avatars.open()
self.ss = self.accountStore.findOrCreate(SubScheduler)
dependency.installOn(self.ss, self.accountStore)
self.origdir = self.accountStore.dbdir
self.destdir = FilePath(self.mktemp())
def createDatabase(s):
"""
Create an account in the given store and install a MailTransferAgent on
both the given store and the substore for that account.
"""
Scheduler(store=s).installOn(s)
loginSystem = LoginSystem(store=s)
loginSystem.installOn(s)
mta = MailTransferAgent(store=s)
mta.installOn(s)
account = loginSystem.addAccount(u'testuser', u'localhost', None)
subStore = account.avatars.open()
SubScheduler(store=subStore).installOn(subStore)
mda = MailTransferAgent(store=subStore)
mda.installOn(subStore)
def setUp(self):
self.clock = Clock()
self.dbdir = filepath.FilePath(self.mktemp())
self.store = Store(self.dbdir)
self.substoreItem = SubStore.createNew(self.store, ['sub'])
self.substore = self.substoreItem.open()
installOn(Scheduler(store=self.store), self.store)
installOn(SubScheduler(store=self.substore), self.substore)
self.scheduler = IScheduler(self.store)
self.subscheduler = IScheduler(self.substore)
self.scheduler.callLater = self.clock.callLater
self.scheduler.now = lambda: Time.fromPOSIXTimestamp(self.clock.
seconds())
self.subscheduler.now = lambda: Time.fromPOSIXTimestamp(self.clock.
seconds())
IService(self.store).startService()
def testProcessingServiceStepsOverErrors(self):
"""
If any processor raises an unexpected exception, the work unit which
was being processed should be marked as having had an error and
processing should move on to the next item. Make sure that this
actually happens when L{BatchProcessingService} is handling those
errors.
"""
BATCH_WORK_UNITS = 3
dbdir = self.mktemp()
st = store.Store(dbdir)
Scheduler(store=st).installOn(st)
source = BatchWorkSource(store=st)
for i in range(BATCH_WORK_UNITS):
BatchWorkItem(store=st)
source.addReliableListener(BrokenReliableListener(store=st), iaxiom.REMOTE)
source.addReliableListener(WorkingReliableListener(store=st), iaxiom.REMOTE)
svc = batch.BatchProcessingService(st, iaxiom.REMOTE)
task = svc.step()
# Loop 6 (BATCH_WORK_UNITS * 2) times - three items times two
# listeners, it should not take any more than six iterations to
# completely process all work.
for i in xrange(BATCH_WORK_UNITS * 2):
task.next()
self.assertEquals(
len(log.flushErrors(BrokenException)),
BATCH_WORK_UNITS)
self.assertEquals(
st.query(BatchWorkItem, BatchWorkItem.value == u"processed").count(),
BATCH_WORK_UNITS)
def createDatabase(store):
scheduler = Scheduler(store=store)
installOn(scheduler, store)
installOn(
_SubSchedulerParentHook(store=store,
loginAccount=SubStore(store=store)), store)
class BatchTestCase(unittest.TestCase):
def setUp(self):
self.procType = batch.processor(TestWorkUnit)
self.store = store.Store()
self.scheduler = Scheduler(store=self.store)
self.scheduler.installOn(self.store)
def testItemTypeCreation(self):
"""
Test that processors for a different Item types can be
created, that they are valid Item types themselves, and that
repeated calls return the same object when appropriate.
"""
procB = batch.processor(TestWorkUnit)
self.assertIdentical(self.procType, procB)
procC = batch.processor(ExtraUnit)
self.failIfIdentical(procB, procC)
self.failIfEqual(procB.typeName, procC.typeName)
def testInstantiation(self):
"""
Test that a batch processor can be instantiated and added to a
database, and that it can be retrieved in the usual ways.
"""
proc = self.procType(store=self.store)
self.assertIdentical(self.store.findUnique(self.procType), proc)
def testListenerlessProcessor(self):
"""
Test that a batch processor can be stepped even if it has no
listeners, and that it correctly reports it has no work to do.
"""
proc = self.procType(store=self.store)
self.failIf(proc.step(), "expected no more work to be reported, some was")
TestWorkUnit(store=self.store, information=0)
self.failIf(proc.step(), "expected no more work to be reported, some was")
def testListeners(self):
"""
Test that items can register or unregister their interest in a
processor's batch of items.
"""
proc = self.procType(store=self.store)
listenerA = WorkListener(store=self.store)
listenerB = WorkListener(store=self.store)
self.assertEquals(list(proc.getReliableListeners()), [])
proc.addReliableListener(listenerA)
self.assertEquals(list(proc.getReliableListeners()), [listenerA])
proc.addReliableListener(listenerB)
expected = [listenerA, listenerB]
listeners = list(proc.getReliableListeners())
self.assertEquals(sorted(expected), sorted(listeners))
proc.removeReliableListener(listenerA)
self.assertEquals(list(proc.getReliableListeners()), [listenerB])
proc.removeReliableListener(listenerB)
self.assertEquals(list(proc.getReliableListeners()), [])
def testBasicProgress(self):
"""
Test that when a processor is created and given a chance to
run, it completes some work.
"""
processedItems = []
def listener(item):
processedItems.append(item.information)
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store, listener=listener)
proc.addReliableListener(listener)
self.assertEquals(processedItems, [])
self.failIf(proc.step(), "expected no work to be reported, some was")
self.assertEquals(processedItems, [])
for i in range(3):
TestWorkUnit(store=self.store, information=i)
ExtraUnit(store=self.store, unformashun=unicode(-i))
self.failUnless(proc.step(), "expected more work to be reported, none was")
self.assertEquals(processedItems, [0])
self.failUnless(proc.step(), "expected more work to be reported, none was")
self.assertEquals(processedItems, [0, 1])
self.failIf(proc.step(), "expected no more work to be reported, some was")
self.assertEquals(processedItems, [0, 1, 2])
self.failIf(proc.step(), "expected no more work to be reported, some was")
self.assertEquals(processedItems, [0, 1, 2])
def testProgressAgainstExisting(self):
"""
Test that when a processor is created when work units exist
already, it works backwards to notify its listener of all
those existing work units. Also test that work units created
after the processor are also handled.
"""
processedItems = []
def listener(item):
processedItems.append(item.information)
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store, listener=listener)
for i in range(3):
TestWorkUnit(store=self.store, information=i)
proc.addReliableListener(listener)
self.assertEquals(processedItems, [])
self.failUnless(proc.step(), "expected more work to be reported, none was")
self.assertEquals(processedItems, [2])
self.failUnless(proc.step(), "expected more work to be reported, none was")
self.assertEquals(processedItems, [2, 1])
self.failIf(proc.step(), "expected no more work to be reported, some was")
self.assertEquals(processedItems, [2, 1, 0])
self.failIf(proc.step(), "expected no more work to be reported, some was")
self.assertEquals(processedItems, [2, 1, 0])
for i in range(3, 6):
TestWorkUnit(store=self.store, information=i)
self.failUnless(proc.step(), "expected more work to be reported, none was")
self.assertEquals(processedItems, [2, 1, 0, 3])
self.failUnless(proc.step(), "expected more work to be reported, none was")
self.assertEquals(processedItems, [2, 1, 0, 3, 4])
self.failIf(proc.step(), "expected no more work to be reported, some was")
self.assertEquals(processedItems, [2, 1, 0, 3, 4, 5])
self.failIf(proc.step(), "expected no more work to be reported, some was")
self.assertEquals(processedItems, [2, 1, 0, 3, 4, 5])
def testBrokenListener(self):
"""
Test that if a listener's processItem method raises an
exception, processing continues beyond that item and that an
error marker is created for that item.
"""
errmsg = "This reliable listener is not very reliable!"
processedItems = []
def listener(item):
if item.information == 1:
raise RuntimeError(errmsg)
processedItems.append(item.information)
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store, listener=listener)
proc.addReliableListener(listener)
# Make some work, step the processor, and fake the error handling
# behavior the Scheduler actually provides.
for i in range(3):
TestWorkUnit(store=self.store, information=i)
try:
proc.step()
except batch._ProcessingFailure:
proc.timedEventErrorHandler(
(u"Oh crap, I do not have a TimedEvent, "
"I sure hope that never becomes a problem."),
failure.Failure())
self.assertEquals(processedItems, [0, 2])
errors = list(proc.getFailedItems())
self.assertEquals(len(errors), 1)
self.assertEquals(errors[0][0], listener)
self.assertEquals(errors[0][1].information, 1)
loggedErrors = log.flushErrors(RuntimeError)
self.assertEquals(len(loggedErrors), 1)
self.assertEquals(loggedErrors[0].getErrorMessage(), errmsg)
def testMultipleListeners(self):
"""
Test that a single batch processor with multiple listeners
added at different times delivers each item to each listener.
"""
processedItemsA = []
def listenerA(item):
processedItemsA.append(item.information)
processedItemsB = []
def listenerB(item):
processedItemsB.append(item.information)
proc = self.procType(store=self.store)
for i in range(2):
TestWorkUnit(store=self.store, information=i)
firstListener = WorkListener(store=self.store, listener=listenerA)
proc.addReliableListener(firstListener)
for i in range(2, 4):
TestWorkUnit(store=self.store, information=i)
secondListener = WorkListener(store=self.store, listener=listenerB)
proc.addReliableListener(secondListener)
for i in range(4, 6):
TestWorkUnit(store=self.store, information=i)
for i in range(100):
if not proc.step():
break
else:
self.fail("Processing loop took too long")
self.assertEquals(
processedItemsA, [2, 3, 4, 5, 1, 0])
self.assertEquals(
processedItemsB, [4, 5, 3, 2, 1, 0])
def testRepeatedAddListener(self):
"""
Test that adding the same listener repeatedly has the same
effect as adding it once.
"""
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store)
proc.addReliableListener(listener)
proc.addReliableListener(listener)
self.assertEquals(list(proc.getReliableListeners()), [listener])
def testSuperfluousItemAddition(self):
"""
Test the addItem method for work which would have been done already,
and so for which addItem should therefore be a no-op.
"""
processedItems = []
def listener(item):
processedItems.append(item.information)
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store, listener=listener)
# Create a couple items so there will be backwards work to do.
one = TestWorkUnit(store=self.store, information=0)
two = TestWorkUnit(store=self.store, information=1)
rellist = proc.addReliableListener(listener)
# Create a couple more items so there will be some forwards work to do.
three = TestWorkUnit(store=self.store, information=2)
four = TestWorkUnit(store=self.store, information=3)
# There are only two regions at this point - work behind and work
# ahead; no work has been done yet, so there's no region in between.
# Add items behind and ahead of the point; these should not result in
# any explicit tracking items, since they would have been processed in
# due course anyway.
rellist.addItem(two)
rellist.addItem(three)
for i in range(100):
if not proc.step():
break
else:
self.fail("Processing loop took too long")
self.assertEquals(processedItems, [2, 3, 1, 0])
def testReprocessItemAddition(self):
"""
Test the addItem method for work which is within the bounds of work
already done, and so which would not have been processed without the
addItem call.
"""
processedItems = []
def listener(item):
processedItems.append(item.information)
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store, listener=listener)
rellist = proc.addReliableListener(listener)
one = TestWorkUnit(store=self.store, information=0)
two = TestWorkUnit(store=self.store, information=1)
three = TestWorkUnit(store=self.store, information=2)
for i in range(100):
if not proc.step():
break
else:
self.fail("Processing loop took too long")
self.assertEquals(processedItems, range(3))
# Now that we have processed some items, re-add one of those items to
# be re-processed and make sure it actually does get passed to the
# listener again.
processedItems = []
rellist.addItem(two)
for i in xrange(100):
if not proc.step():
break
else:
self.fail("Processing loop took too long")
self.assertEquals(processedItems, [1])
def test_processorStartsUnscheduled(self):
"""
Test that when a processor is first created, it is not scheduled to
perform any work.
"""
proc = self.procType(store=self.store)
self.assertIdentical(proc.scheduled, None)
self.assertEquals(
list(self.scheduler.scheduledTimes(proc)),
[])
def test_itemAddedIgnoredWithoutListeners(self):
"""
Test that if C{itemAdded} is called while the processor is idle but
there are no listeners, the processor does not schedule itself to do
any work.
"""
proc = self.procType(store=self.store)
proc.itemAdded()
self.assertEqual(proc.scheduled, None)
self.assertEquals(
list(self.scheduler.scheduledTimes(proc)),
[])
def test_itemAddedSchedulesProcessor(self):
"""
Test that if C{itemAdded} is called while the processor is idle and
there are listeners, the processor does schedules itself to do some
work.
"""
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store)
proc.addReliableListener(listener)
# Get rid of the scheduler state that addReliableListener call just
# created.
proc.scheduled = None
self.scheduler.unscheduleAll(proc)
proc.itemAdded()
self.failIfEqual(proc.scheduled, None)
self.assertEquals(
list(self.scheduler.scheduledTimes(proc)),
[proc.scheduled])
def test_addReliableListenerSchedulesProcessor(self):
"""
Test that if C{addReliableListener} is called while the processor is
idle, the processor schedules itself to do some work.
"""
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store)
proc.addReliableListener(listener)
self.failIfEqual(proc.scheduled, None)
self.assertEquals(
list(self.scheduler.scheduledTimes(proc)),
[proc.scheduled])
def test_itemAddedWhileScheduled(self):
"""
Test that if C{itemAdded} is called when the processor is already
scheduled to run, the processor remains scheduled to run at the same
time.
"""
proc = self.procType(store=self.store)
listener = WorkListener(store=self.store)
proc.addReliableListener(listener)
when = proc.scheduled
proc.itemAdded()
self.assertEquals(proc.scheduled, when)
self.assertEquals(
list(self.scheduler.scheduledTimes(proc)),
[proc.scheduled])
def test_addReliableListenerWhileScheduled(self):
"""
Test that if C{addReliableListener} is called when the processor is
already scheduled to run, the processor remains scheduled to run at the
same time.
"""
proc = self.procType(store=self.store)
listenerA = WorkListener(store=self.store)
proc.addReliableListener(listenerA)
when = proc.scheduled
listenerB = WorkListener(store=self.store)
proc.addReliableListener(listenerB)
self.assertEquals(proc.scheduled, when)
self.assertEquals(
list(self.scheduler.scheduledTimes(proc)),
[proc.scheduled])
def test_processorIdlesWhenCaughtUp(self):
"""
Test that the C{run} method of the processor returns C{None} when it
has done all the work it needs to do, thus unscheduling the processor.
"""
proc = self.procType(store=self.store)
self.assertIdentical(proc.run(), None)
def createDatabase(s):
Scheduler(store=s).installOn(s)
MessageSource(store=s)
tc = Catalog(store=s)
rfp = RuleFilteringPowerup(store=s, tagCatalog=tc).installOn(s)
def createDatabase(s):
Scheduler(store=s).installOn(s)
MessageSource(store=s)
Filter(store=s).installOn(s)
def setUp(self):
self.procType = batch.processor(TestWorkUnit)
self.store = store.Store()
self.scheduler = Scheduler(store=self.store)
self.scheduler.installOn(self.store)
def setUp(self):
# self.storePath = self.mktemp()
self.store = Store()
Scheduler(store=self.store).installOn(self.store)
IService(self.store).startService()
def createDatabase(store):
installOn(Scheduler(store=store), store)