def test_scheduler(self):
"""
Test that the ordering and timing of scheduled calls is correct.
"""
# create 3 timed events. the first one fires. the second one fires,
# then reschedules itself. the third one should never fire because the
# reactor is shut down first. assert that the first and second fire
# only once, and that the third never fires.
s = self.store
t1 = TestEvent(testCase=self,
name=u't1',
store=s, runAgain=None)
t2 = TestEvent(testCase=self,
name=u't2',
store=s, runAgain=2)
t3 = TestEvent(testCase=self,
name=u't3',
store=s, runAgain=None)
now = self.now()
self.ts = [t1, t2, t3]
S = IScheduler(s)
# Schedule them out of order to make sure behavior doesn't
# depend on tasks arriving in soonest-to-latest order.
S.schedule(t2, now + timedelta(seconds=3))
S.schedule(t1, now + timedelta(seconds=1))
S.schedule(t3, now + timedelta(seconds=100))
self.clock.pump([2, 2, 2])
self.assertEqual(t1.runCount, 1)
self.assertEqual(t2.runCount, 2)
self.assertEqual(t3.runCount, 0)
def testScheduledErrorWithHandler(self):
S = IScheduler(self.store)
now = Time()
spec = SpecialErrorHandler(store=self.store)
S.schedule(spec, now)
d = Deferred()
te = TestEvent(store=self.store,
testCase=self,
name=u't1',
maxRunCount=1,
runAgain=None,
winner=True,
deferred=d)
self.te = te # don't gc the deferred
now2 = Time()
S.schedule(te, now2)
self.assertEquals(self.store.query(TimedEventFailureLog).count(), 0)
def later(result):
errs = log.flushErrors(SpecialError)
self.assertEquals(len(errs), 1)
self.assertEquals(
self.store.query(TimedEventFailureLog).count(), 0)
self.failUnless(spec.procd)
self.failIf(spec.broken)
return d.addCallback(later)
def testUnscheduling(self):
"""
Test the unscheduleFirst method of the scheduler.
"""
now = Time()
d = Deferred()
sch = IScheduler(self.store)
t1 = TestEvent(testCase=self,
name=u't1',
store=self.store,
maxRunCount=0)
t2 = TestEvent(testCase=self,
name=u't2',
store=self.store,
maxRunCount=1,
runAgain=None,
winner=True,
deferred=d)
# Make sure the inmemory attributes hang around
self.ts = [t1, t2]
sch.schedule(t1, now + timedelta(milliseconds=100))
sch.schedule(t2, now + timedelta(milliseconds=200))
sch.unscheduleFirst(t1)
return d
def _scheduleMePlease(self):
"""
This queue needs to have its run() method invoked at some point in the
future. Tell the dependent scheduler to schedule it if it isn't
already pending execution.
"""
sched = IScheduler(self.store)
if len(list(sched.scheduledTimes(self))) == 0:
sched.schedule(self, sched.now())
def _scheduleMePlease(self):
"""
This queue needs to have its run() method invoked at some point in the
future. Tell the dependent scheduler to schedule it if it isn't
already pending execution.
"""
sched = IScheduler(self.store)
if len(list(sched.scheduledTimes(self))) == 0:
sched.schedule(self, sched.now())
def _schedule(self, when):
"""
Ensure that this hook is scheduled to run at or before C{when}.
"""
sched = IScheduler(self.store)
for scheduledAt in sched.scheduledTimes(self):
if when < scheduledAt:
sched.reschedule(self, scheduledAt, when)
break
else:
sched.schedule(self, when)
def _schedule(self, when):
"""
Ensure that this hook is scheduled to run at or before C{when}.
"""
sched = IScheduler(self.store)
for scheduledAt in sched.scheduledTimes(self):
if when < scheduledAt:
sched.reschedule(self, scheduledAt, when)
break
else:
sched.schedule(self, when)
class PastesAPIResource(Resource):
"""
Resource for the "pastes" API.
I{POST}ing to this resource will create a new L{Paste} item. I{GET}ting
a child of this resource will look a L{Paste} item up by its I{name}
attribute.
"""
def __init__(self, store):
self._store = store
self._scheduler = IScheduler(self._store)
self._chain = japaneseChain()
Resource.__init__(self)
def _generateName(self, n=8):
"""
Generate a paste name.
@type n: L{int}
@param n: Length of the name to generate.
@rtype: L{unicode}
"""
return u''.join(itertools.islice(self._chain, n))
def _createPaste(self, content, languageHint=None):
p = Paste(
store=self._store,
name=self._generateName(),
content=content,
languageHint=languageHint)
# <k4y> so what's a good paste lifetime?
# * mithrandi picks "6 days" out of a musical hat
# <k4y> what's it playing?
# <mithrandi> DJ Shadow - Six Days
expiryDate = p.created + timedelta(days=6)
self._scheduler.schedule(p, expiryDate)
return p
def render_POST(self, request):
data = json.load(request.content)
return self._createPaste(**data).toJSON()
def getChild(self, path, request):
try:
paste = Paste.findByName(self._store, path.decode('ascii'))
except ItemNotFound:
return NoResource('No such paste')
else:
return Data(paste.toJSON(), 'application/json')
def test_unscheduling(self):
"""
Test the unscheduleFirst method of the scheduler.
"""
sch = IScheduler(self.store)
t1 = TestEvent(testCase=self, name=u't1', store=self.store)
t2 = TestEvent(testCase=self, name=u't2', store=self.store, runAgain=None)
sch.schedule(t1, self.now() + timedelta(seconds=1))
sch.schedule(t2, self.now() + timedelta(seconds=2))
sch.unscheduleFirst(t1)
self.clock.advance(3)
self.assertEquals(t1.runCount, 0)
self.assertEquals(t2.runCount, 1)
class PastesAPIResource(Resource):
"""
Resource for the "pastes" API.
I{POST}ing to this resource will create a new L{Paste} item. I{GET}ting
a child of this resource will look a L{Paste} item up by its I{name}
attribute.
"""
def __init__(self, store):
self._store = store
self._scheduler = IScheduler(self._store)
self._chain = japaneseChain()
Resource.__init__(self)
def _generateName(self, n=8):
"""
Generate a paste name.
@type n: L{int}
@param n: Length of the name to generate.
@rtype: L{unicode}
"""
return u''.join(itertools.islice(self._chain, n))
def _createPaste(self, content, languageHint=None):
p = Paste(store=self._store,
name=self._generateName(),
content=content,
languageHint=languageHint)
# <k4y> so what's a good paste lifetime?
# * mithrandi picks "6 days" out of a musical hat
# <k4y> what's it playing?
# <mithrandi> DJ Shadow - Six Days
expiryDate = p.created + timedelta(days=6)
self._scheduler.schedule(p, expiryDate)
return p
def render_POST(self, request):
data = json.load(request.content)
return self._createPaste(**data).toJSON()
def getChild(self, path, request):
try:
paste = Paste.findByName(self._store, path.decode('ascii'))
except ItemNotFound:
return NoResource('No such paste')
else:
return Data(paste.toJSON(), 'application/json')
def test_scheduledTimesDuringRun(self):
"""
L{Scheduler.scheduledTimes} should not include scheduled times that have
already triggered.
"""
futureTimes = []
scheduler = IScheduler(self.store)
runner = HookRunner(store=self.store,
hook=lambda self: futureTimes.append(
list(scheduler.scheduledTimes(self))))
then = self.now() + timedelta(seconds=1)
scheduler.schedule(runner, self.now())
scheduler.schedule(runner, then)
self.clock.advance(1)
self.assertEquals(futureTimes, [[then], []])
def test_scheduledTimesDuringRun(self):
"""
L{Scheduler.scheduledTimes} should not include scheduled times that have
already triggered.
"""
futureTimes = []
scheduler = IScheduler(self.store)
runner = HookRunner(
store=self.store,
hook=lambda self: futureTimes.append(
list(scheduler.scheduledTimes(self))))
then = self.now() + timedelta(seconds=1)
scheduler.schedule(runner, self.now())
scheduler.schedule(runner, then)
self.clock.advance(1)
self.assertEquals(futureTimes, [[then], []])
def testBasicScheduledError(self):
S = IScheduler(self.store)
S.schedule(NotActuallyRunnable(store=self.store), self.now())
te = TestEvent(store=self.store, testCase=self,
name=u't1', runAgain=None)
S.schedule(te, self.now() + timedelta(seconds=1))
self.assertEquals(
self.store.query(TimedEventFailureLog).count(), 0)
self.clock.advance(3)
self.assertEquals(te.runCount, 1)
errs = self.flushLoggedErrors(AttributeError)
self.assertEquals(len(errs), 1)
self.assertEquals(self.store.query(TimedEventFailureLog).count(), 1)
def _doTestScheduler(self, s):
# create 3 timed events. the first one fires. the second one fires,
# then reschedules itself. the third one should never fire because the
# reactor is shut down first. assert that the first and second fire
# only once, and that the third never fires.
d = Deferred()
interval = 30
t1 = TestEvent(testCase=self,
name=u't1',
store=s,
maxRunCount=1,
runAgain=None,
deferred=d)
t2 = TestEvent(testCase=self,
name=u't2',
store=s,
maxRunCount=2,
runAgain=interval,
deferred=d,
winner=True)
t3 = TestEvent(testCase=self,
name=u't3',
store=s,
maxRunCount=0,
runAgain=None,
deferred=d)
now = Time()
self.ts = [t1, t2, t3]
S = IScheduler(s)
# Schedule them out of order to make sure behavior doesn't
# depend on tasks arriving in soonest-to-latest order.
S.schedule(t2, now + timedelta(milliseconds=interval * 2))
S.schedule(t1, now + timedelta(milliseconds=interval * 1))
S.schedule(t3, now + timedelta(milliseconds=interval * 100))
return d
def testScheduledErrorWithHandler(self):
S = IScheduler(self.store)
spec = SpecialErrorHandler(store=self.store)
S.schedule(spec, self.now())
te = TestEvent(store=self.store, testCase=self,
name=u't1', runAgain=None)
S.schedule(te, self.now() + timedelta(seconds=1))
self.assertEquals(
self.store.query(TimedEventFailureLog).count(), 0)
self.clock.advance(3)
self.assertEquals(te.runCount, 1)
errs = self.flushLoggedErrors(SpecialError)
self.assertEquals(len(errs), 1)
self.assertEquals(self.store.query(TimedEventFailureLog).count(), 0)
self.failUnless(spec.procd)
self.failIf(spec.broken)
def test_deletedRunnable(self):
"""
Verify that if a scheduled item is deleted,
L{TimedEvent.invokeRunnable} just deletes the L{TimedEvent} without
raising an exception.
"""
now = self.now()
scheduler = IScheduler(self.store)
runnable = TestEvent(store=self.store, name=u'Only event')
scheduler.schedule(runnable, now)
runnable.deleteFromStore()
# Invoke it manually to avoid timing complexity.
timedEvent = self.store.findUnique(TimedEvent,
TimedEvent.runnable == runnable)
timedEvent.invokeRunnable()
self.assertEqual(
self.store.findUnique(TimedEvent,
TimedEvent.runnable == runnable,
default=None), None)
def test_deletedRunnable(self):
"""
Verify that if a scheduled item is deleted,
L{TimedEvent.invokeRunnable} just deletes the L{TimedEvent} without
raising an exception.
"""
now = self.now()
scheduler = IScheduler(self.store)
runnable = TestEvent(store=self.store, name=u'Only event')
scheduler.schedule(runnable, now)
runnable.deleteFromStore()
# Invoke it manually to avoid timing complexity.
timedEvent = self.store.findUnique(
TimedEvent, TimedEvent.runnable == runnable)
timedEvent.invokeRunnable()
self.assertEqual(
self.store.findUnique(
TimedEvent,
TimedEvent.runnable == runnable,
default=None),
None)
def testBasicScheduledError(self):
S = IScheduler(self.store)
now = Time()
S.schedule(NotActuallyRunnable(store=self.store), now)
d = Deferred()
te = TestEvent(store=self.store,
testCase=self,
name=u't1',
maxRunCount=1,
runAgain=None,
winner=True,
deferred=d)
self.te = te # don't gc the deferred
now2 = Time()
S.schedule(te, now2)
self.assertEquals(self.store.query(TimedEventFailureLog).count(), 0)
def later(result):
errs = log.flushErrors(AttributeError)
self.assertEquals(len(errs), 1)
self.assertEquals(
self.store.query(TimedEventFailureLog).count(), 1)
return d.addCallback(later)
def test_inspection(self):
"""
Test that the L{scheduledTimes} method returns an iterable of all the
times at which a particular item is scheduled to run.
"""
now = self.now() + timedelta(seconds=1)
off = timedelta(seconds=3)
sch = IScheduler(self.store)
runnable = TestEvent(store=self.store, name=u'Only event')
sch.schedule(runnable, now)
sch.schedule(runnable, now + off)
sch.schedule(runnable, now + off + off)
self.assertEquals(list(sch.scheduledTimes(runnable)),
[now, now + off, now + off + off])
def test_inspection(self):
"""
Test that the L{scheduledTimes} method returns an iterable of all the
times at which a particular item is scheduled to run.
"""
now = self.now() + timedelta(seconds=1)
off = timedelta(seconds=3)
sch = IScheduler(self.store)
runnable = TestEvent(store=self.store, name=u'Only event')
sch.schedule(runnable, now)
sch.schedule(runnable, now + off)
sch.schedule(runnable, now + off + off)
self.assertEquals(
list(sch.scheduledTimes(runnable)),
[now, now + off, now + off + off])
class SubStoreSchedulerReentrancy(TestCase):
"""
Test re-entrant scheduling calls on an item run by a SubScheduler.
"""
def setUp(self):
self.clock = Clock()
self.dbdir = filepath.FilePath(self.mktemp())
self.store = Store(self.dbdir)
self.substoreItem = SubStore.createNew(self.store, ['sub'])
self.substore = self.substoreItem.open()
self.scheduler = IScheduler(self.store)
self.subscheduler = IScheduler(self.substore)
self.scheduler.callLater = self.clock.callLater
self.scheduler.now = lambda: Time.fromPOSIXTimestamp(self.clock.seconds())
self.subscheduler.now = lambda: Time.fromPOSIXTimestamp(self.clock.seconds())
IService(self.store).startService()
def tearDown(self):
return IService(self.store).stopService()
def _scheduleRunner(self, now, offset):
scheduledAt = Time.fromPOSIXTimestamp(now + offset)
rescheduleFor = Time.fromPOSIXTimestamp(now + offset + 10)
runnable = ScheduleCallingItem(store=self.substore, rescheduleFor=rescheduleFor)
self.subscheduler.schedule(runnable, scheduledAt)
return runnable
def testSchedule(self):
"""
Test the schedule method, as invoked from the run method of an item
being run by the subscheduler.
"""
now = self.clock.seconds()
runnable = self._scheduleRunner(now, 10)
self.clock.advance(11)
self.assertEqual(
list(self.subscheduler.scheduledTimes(runnable)),
[Time.fromPOSIXTimestamp(now + 20)])
hook = self.store.findUnique(
_SubSchedulerParentHook,
_SubSchedulerParentHook.subStore == self.substoreItem)
self.assertEqual(
list(self.scheduler.scheduledTimes(hook)),
[Time.fromPOSIXTimestamp(now + 20)])
def testScheduleWithLaterTimedEvents(self):
"""
Like L{testSchedule}, but use a SubScheduler which has pre-existing
TimedEvents which are beyond the new runnable's scheduled time (to
trigger the reschedule-using code-path in
_SubSchedulerParentHook._schedule).
"""
now = self.clock.seconds()
when = Time.fromPOSIXTimestamp(now + 30)
null = NullRunnable(store=self.substore)
self.subscheduler.schedule(null, when)
runnable = self._scheduleRunner(now, 10)
self.clock.advance(11)
self.assertEqual(
list(self.subscheduler.scheduledTimes(runnable)),
[Time.fromPOSIXTimestamp(now + 20)])
self.assertEqual(
list(self.subscheduler.scheduledTimes(null)),
[Time.fromPOSIXTimestamp(now + 30)])
hook = self.store.findUnique(
_SubSchedulerParentHook,
_SubSchedulerParentHook.subStore == self.substoreItem)
self.assertEqual(
list(self.scheduler.scheduledTimes(hook)),
[Time.fromPOSIXTimestamp(20)])
def testScheduleWithEarlierTimedEvents(self):
"""
Like L{testSchedule}, but use a SubScheduler which has pre-existing
TimedEvents which are before the new runnable's scheduled time.
"""
now = self.clock.seconds()
when = Time.fromPOSIXTimestamp(now + 15)
null = NullRunnable(store=self.substore)
self.subscheduler.schedule(null, when)
runnable = self._scheduleRunner(now, 10)
self.clock.advance(11)
self.assertEqual(
list(self.subscheduler.scheduledTimes(runnable)),
[Time.fromPOSIXTimestamp(now + 20)])
self.assertEqual(
list(self.subscheduler.scheduledTimes(null)),
[Time.fromPOSIXTimestamp(now + 15)])
hook = self.store.findUnique(
_SubSchedulerParentHook,
_SubSchedulerParentHook.subStore == self.substoreItem)
self.assertEqual(
list(self.scheduler.scheduledTimes(hook)),
[Time.fromPOSIXTimestamp(now + 15)])
def testMultipleEventsPerTick(self):
"""
Test running several runnables in a single tick of the subscheduler.
"""
now = self.clock.seconds()
runnables = [
self._scheduleRunner(now, 10),
self._scheduleRunner(now, 11),
self._scheduleRunner(now, 12)]
self.clock.advance(13)
for n, runnable in enumerate(runnables):
self.assertEqual(
list(self.subscheduler.scheduledTimes(runnable)),
[Time.fromPOSIXTimestamp(now + n + 20)])
hook = self.store.findUnique(
_SubSchedulerParentHook,
_SubSchedulerParentHook.subStore == self.substoreItem)
self.assertEqual(
list(self.scheduler.scheduledTimes(hook)),
[Time.fromPOSIXTimestamp(now + 20)])
def run(self):
scheduler = IScheduler(self.store)
scheduler.schedule(self, self.rescheduleFor)
self.ran = True
class SubStoreMigrationTestCase(unittest.TestCase):
IMPORTANT_VALUE = 159
localpart = 'testuser'
domain = 'example.com'
def setUp(self):
self.dbdir = FilePath(self.mktemp())
self.store = Store(self.dbdir)
self.ls = userbase.LoginSystem(store=self.store)
self.scheduler = IScheduler(self.store)
self.account = self.ls.addAccount(
self.localpart, self.domain, 'PASSWORD')
self.accountStore = self.account.avatars.open()
self.ss = IScheduler(self.accountStore)
self.origdir = self.accountStore.dbdir
self.destdir = FilePath(self.mktemp())
def test_extraction(self):
"""
Ensure that user store extraction works correctly,
particularly in the presence of timed events.
"""
thing = ThingThatMovesAround(store=self.accountStore,
superValue=self.IMPORTANT_VALUE)
self.ss.schedule(thing, Time() + datetime.timedelta(days=1))
self.test_noTimedEventsExtraction()
def test_noTimedEventsExtraction(self):
"""
Ensure that user store extraction works correctly if no timed
events are present.
"""
userbase.extractUserStore(self.account, self.destdir)
self.assertEqual(
self.ls.accountByAddress(self.localpart, self.domain),
None)
self.assertFalse(list(self.store.query(SubStore, SubStore.storepath == self.origdir)))
self.origdir.restat(False)
self.assertFalse(self.origdir.exists())
self.assertFalse(list(self.store.query(_SubSchedulerParentHook)))
def test_noTimedEventsInsertion(self):
"""
Test that inserting a user store succeeds if it contains no
timed events.
"""
self.test_noTimedEventsExtraction()
self._testInsertion()
def test_insertion(self, _deleteDomainDirectory=False):
"""
Test that inserting a user store succeeds and that the right
items are placed in the site store as a result.
"""
self.test_extraction()
self._testInsertion(_deleteDomainDirectory)
insertedStore = self.ls.accountByAddress(self.localpart,
self.domain).avatars.open()
self.assertEqual(
insertedStore.findUnique(ThingThatMovesAround).superValue,
self.IMPORTANT_VALUE)
siteStoreSubRef = self.store.getItemByID(insertedStore.idInParent)
ssph = self.store.findUnique(_SubSchedulerParentHook,
_SubSchedulerParentHook.subStore == siteStoreSubRef,
default=None)
self.assertTrue(ssph)
self.assertTrue(self.store.findUnique(TimedEvent,
TimedEvent.runnable == ssph))
def _testInsertion(self, _deleteDomainDirectory=False):
"""
Helper method for inserting a user store.
"""
if _deleteDomainDirectory:
self.store.filesdir.child('account').child(self.domain).remove()
userbase.insertUserStore(self.store, self.destdir)
def test_insertionWithNoDomainDirectory(self):
"""
Test that inserting a user store succeeds even if it is the
first one in that domain to be inserted.
"""
self.test_insertion(True)
