def upgradeProcessor1to2(oldProcessor):
"""
Batch processors stopped polling at version 2, so they no longer needed the
idleInterval attribute. They also gained a scheduled attribute which
tracks their interaction with the scheduler. Since they stopped polling,
we also set them up as a timed event here to make sure that they don't
silently disappear, never to be seen again: running them with the scheduler
gives them a chance to figure out what's up and set up whatever other state
they need to continue to run.
Since this introduces a new dependency of all batch processors on a powerup
for the IScheduler, install a Scheduler or a SubScheduler if one is not
already present.
"""
newProcessor = oldProcessor.upgradeVersion(
oldProcessor.typeName, 1, 2,
busyInterval=oldProcessor.busyInterval)
newProcessor.scheduled = extime.Time()
s = newProcessor.store
sch = iaxiom.IScheduler(s, None)
if sch is None:
if s.parent is None:
# Only site stores have no parents.
sch = Scheduler(store=s)
else:
# Substores get subschedulers.
sch = SubScheduler(store=s)
installOn(sch, s)
# And set it up to run.
sch.schedule(newProcessor, newProcessor.scheduled)
return newProcessor
def itemAdded(self):
"""
Called to indicate that a new item of the type monitored by this batch
processor is being added to the database.
If this processor is not already scheduled to run, this will schedule
it. It will also start the batch process if it is not yet running and
there are any registered remote listeners.
"""
localCount = self.store.query(
_ReliableListener,
attributes.AND(_ReliableListener.processor == self,
_ReliableListener.style == iaxiom.LOCAL),
limit=1).count()
remoteCount = self.store.query(
_ReliableListener,
attributes.AND(_ReliableListener.processor == self,
_ReliableListener.style == iaxiom.REMOTE),
limit=1).count()
if localCount and self.scheduled is None:
self.scheduled = extime.Time()
iaxiom.IScheduler(self.store).schedule(self, self.scheduled)
if remoteCount:
batchService = iaxiom.IBatchService(self.store, None)
if batchService is not None:
batchService.start()
def stoppedRunning(self):
if self.grabber is None:
return
self.grabber.running = False
if self._transient:
iaxiom.IScheduler(self.grabber.store).reschedule(
self.grabber, self.grabber.scheduled, extime.Time())
self.grabber = None
def delete(self):
iaxiom.IScheduler(self.store).unscheduleAll(self)
if self.running:
if self.protocol is not None:
self.protocol.stop()
self.protocol.grabber = None
else:
self.connector.disconnect()
self.deleteFromStore()
def test_delete(self):
"""
L{POP3Grabber.delete} unschedules the grabber.
"""
store = self.grabber.store
iaxiom.IScheduler(store).schedule(self.grabber, extime.Time())
self.grabber.delete()
# Can't query for the TimedEvent directly, but we know nothing *else*
# was scheduled either.
self.assertEqual(
[], list(store.query(scheduler.TimedEvent)))
def setUp(self):
"""
Create a grabber in a user store.
"""
self.siteStore = store.Store()
self.subStore = substore.SubStore.createNew(self.siteStore, ['grabber'])
self.userStore = self.subStore.open()
self.scheduler = iaxiom.IScheduler(self.userStore)
self.grabberItem = grabber.POP3Grabber(
store=self.userStore, username=u"alice", domain=u"example.com",
password=u"secret", running=True,
config=grabber.GrabberConfiguration(store=self.userStore))
self.grabberItem.scheduled = extime.Time()
self.scheduler.schedule(self.grabberItem, self.grabberItem.scheduled)
def addGrabber(self, username, password, domain, ssl):
# DO IT
if ssl:
port = 995
else:
port = 110
pg = POP3Grabber(store=self.store,
username=username,
password=password,
domain=domain,
port=port,
config=self,
ssl=ssl)
# DO IT *NOW*
iaxiom.IScheduler(self.store).schedule(pg, extime.Time())
def test_addGrabber(self):
"""
L{GrabberConfiguration.addGrabber} creates a new L{POP3Grabber} item
scheduled to run immediately.
"""
siteStore = store.Store()
subStore = substore.SubStore.createNew(siteStore, ['grabber'])
userStore = subStore.open()
scheduler = iaxiom.IScheduler(userStore)
config = grabber.GrabberConfiguration(store=userStore)
config.addGrabber(u"alice", u"secret", u"example.com", False)
grabberItems = list(userStore.query(grabber.POP3Grabber))
self.assertEqual(1, len(grabberItems))
scheduled = list(scheduler.scheduledTimes(grabberItems[0]))
self.assertEqual(1, len(scheduled))
self.assertTrue(scheduled[0] <= extime.Time())
def addReliableListener(self, listener, style=iaxiom.LOCAL):
"""
Add the given Item to the set which will be notified of Items
available for processing.
Note: Each Item is processed synchronously. Adding too many
listeners to a single batch processor will cause the L{step}
method to block while it sends notification to each listener.
@param listener: An Item instance which provides a
C{processItem} method.
@return: An Item representing L{listener}'s persistent tracking state.
"""
existing = self.store.findUnique(
_ReliableListener,
attributes.AND(_ReliableListener.processor == self,
_ReliableListener.listener == listener),
default=None)
if existing is not None:
return existing
for work in self.store.query(self.workUnitType,
sort=self.workUnitType.storeID.descending,
limit=1):
forwardMark = work.storeID
backwardMark = work.storeID + 1
break
else:
forwardMark = 0
backwardMark = 0
if self.scheduled is None:
self.scheduled = extime.Time()
iaxiom.IScheduler(self.store).schedule(self, self.scheduled)
return _ReliableListener(store=self.store,
processor=self,
listener=listener,
forwardMark=forwardMark,
backwardMark=backwardMark,
style=style)
def run(self):
"""
Try to reliably deliver this message. If errors are encountered, try
harder.
"""
sch = iaxiom.IScheduler(self.store)
if self.tries < len(self.RETRANS_TIMES):
# Set things up to try again, if this attempt fails
nextTry = datetime.timedelta(
seconds=self.RETRANS_TIMES[self.tries])
sch.schedule(self, extime.Time() + nextTry)
if not self.running:
self.running = True
self.tries += 1
d = self.sendmail()
d.addCallback(self.mailSent, sch)
d.addErrback(self.failureSending, sch)
d.addErrback(log.err)
def setUp(self):
self.procType = batch.processor(TestWorkUnit)
self.store = store.Store()
self.scheduler = iaxiom.IScheduler(self.store)