def testStopAndCancelWithOneUnderway(self):
"""
Start a dispatch queue of width 2, and send it 3 jobs. Verify that
2 of the jobs are underway. Then stop it before they can complete,
telling it to cancel the underway jobs. The two jobs that were
underway should both be cancelled and returned by the stop method.
The first 2 jobs returned should have state CANCELLED, and the
final one should still be PENDING.
"""
def ok(result):
self.fail('Unexpected success!')
def checkCancel(failure):
self.assertEqual(failure.value.state, Job.CANCELLED)
dq = ResizableDispatchQueue(self.slow, 2)
dq.put(0).addCallbacks(ok, checkCancel)
dq.put(1).addCallbacks(ok, checkCancel)
dq.put(2)
reactor.callLater(0.01, self._testUnderway, dq, set([0, 1]))
pendingJobs = yield task.deferLater(
reactor, 0.1, dq.stop, cancelUnderway=True)
pendingArgs = [p.jobarg for p in pendingJobs]
self.assertEqual([0, 1, 2], sorted(pendingArgs))
self.assertEqual(pendingJobs[0].state, Job.CANCELLED)
self.assertEqual(pendingJobs[1].state, Job.CANCELLED)
self.assertEqual(pendingJobs[2].state, Job.PENDING)
def testReprioritizePending(self):
"""
Reprioritize a pending job.
"""
dq = ResizableDispatchQueue(None, 0)
dq.put('xxx', 5)
pending = dq.pending()
job = pending.pop()
dq.reprioritize(job, 10)
self.assertEqual(10, job.priority)
def testSetWidthToZeroAfterInitiallyNonZeroThenStop(self):
"""
Make sure that a queue whose width is initially non-zero and which
is then set to zero width returns all the added jobs when stopped.
"""
dq = ResizableDispatchQueue(None, 3)
dq.width = 0
dq.put('aaa', 5)
dq.put('bbb', 10)
remaining = yield dq.stop()
self.assertEqual(['aaa', 'bbb'], [job.jobarg for job in remaining])
def testSetWidthToZeroAfterInitiallyNonZero(self):
"""
Make sure that a queue whose width is initially non-zero and which
is then set to zero width does not then begin to process any added
jobs.
"""
dq = ResizableDispatchQueue(None, 3)
dq.width = 0
dq.put('aaa')
dq.put('bbb')
self.assertEqual((0, 2), dq.size())
return dq.stop()
def testCancelRemovesFromQueue(self):
"""
Cancelling a job should remove it from the queue.
"""
def eb(failure):
self.assertEqual(failure.value.state, Job.CANCELLED)
dq = ResizableDispatchQueue(self.slow, 1)
dq.put('a', 1)
deferred = dq.put('b', 2)
deferred.addErrback(eb)
deferred.cancel()
yield task.deferLater(reactor, 0.1, dq.pause)
pendingJobs = dq.pending()
self.assertEqual(pendingJobs, [])
def testReprioritizeNonPendingJob(self):
"""
Reprioritizing a job that is not pending must raise
C{RuntimeError}.
"""
dq = ResizableDispatchQueue(lambda x: x, 1)
job = yield dq.put('a')
self.assertRaises(RuntimeError, dq.reprioritize, job, 10)
def testDeferredFromPutFiresWithCorrectResult(self):
"""
Make sure the C{Deferred} we get back from a put to the queue fires
with the expected result.
"""
dq = ResizableDispatchQueue(lambda x: 2 * x, width=1)
job = yield dq.put(6)
self.assertEqual(job.result, 12)
def testClearQueueCancelPending(self):
"""
When the queue is cleared with cancelPending=True, the pending jobs
should fail and receive a job whose state is CANCELLED.
"""
def ok(result):
self.fail('Unexpected success!')
def checkCancel(failure):
self.assertEqual(failure.value.state, Job.CANCELLED)
dq = ResizableDispatchQueue(None, 2)
dq.put(0).addCallbacks(ok, checkCancel)
dq.put(1).addCallbacks(ok, checkCancel)
self.assertEqual((0, 2), dq.size())
dq.clearQueue(cancelPending=True)
self.assertEqual((0, 0), dq.size())
yield dq.stop()
def testRaiseFailsWithJob(self):
"""
Raising an exception in the job handler should result in a failure
that contains a L{txrdq.job.Job} value.
"""
def raiseException(jobarg):
raise Exception()
dq = ResizableDispatchQueue(raiseException, 1)
d = dq.put('Some argument', 1)
return self.assertFailure(d, Job)
def testRaiseFailsWithCorrectJobArgument(self):
"""
Raising an exception in the job handler should result in a failure
that contains a job with the original job argument.
"""
result = []
def raiseException(jobarg):
raise Exception()
def extractJobarg(failure):
result.append(failure.value.jobarg)
dq = ResizableDispatchQueue(raiseException, 1)
yield dq.put('Some argument', 1).addErrback(extractJobarg)
self.assertEqual(['Some argument'], result)
class AdderCache(DumpingCache):
def __init__(self, cache, queueWidth, endpoint):
super(AdderCache, self).__init__()
self.cache = cache
self.queueWidth = queueWidth
self.endpoint = endpoint
def load(self, cacheFile):
self.rdq = ResizableDispatchQueue(self._addUser, width=self.queueWidth)
self.users = super(AdderCache, self).load(cacheFile)
if self.users is None:
self.users = {}
self.setCache(self.users)
else:
added = [u for u in self.users.values() if u.state == "added"]
notAdded = [u for u in self.users.values() if u.state != "added"]
log.msg("Loaded adder cache: found %d added, %d unadded users" % (len(added), len(notAdded)))
if self.cache.restoreAddQueue:
log.msg("Restoring add queue.")
# Re-queue, in original queue order, any users that are
# marked as having been added.
for user in sorted(notAdded, key=attrgetter("queuedAt")):
log.msg("Restoring %r (previous state %r)" % (user.screenname, user.state))
user.reset()
self.rdq.put(user)
self.clean = False
else:
log.msg("Not restoring formerly queued names.")
# Drop users that were not added last time.
for user in notAdded:
log.msg("Dropping user %r (in state %r)" % (user.screenname, user.state))
del self.users[user.screenname.lower()]
self.clean = False
def __str__(self):
s = ["%d users in adder cache" % len(self.users)]
for key in sorted(self.users.keys()):
s.append(str(self.users[key]))
return "\n".join(s)
def put(self, screenname, nFriends):
screennameLower = screenname.lower()
user = self.users.get(screennameLower)
if user:
user.nFriends = nFriends
user.setState("queued")
else:
user = User(screenname, nFriends)
self.users[screennameLower] = user
log.msg("Adding screenname %r to request queue." % screenname)
self.clean = False
self.rdq.put(user)
def _addUser(self, user):
def _added(result):
user.setState("added")
self.clean = False
return result
def _failed(fail):
self.clean = False
if fail.check(ftwitter.Canceled):
# The state has been changed to canceled below.
assert user.canceled()
log.msg("Addition of user %r canceled." % user.screenname)
else:
user.setState("failed")
log.msg("Failed to add %r: %s" % (user.screenname, fail))
log.msg("User %r received from request queue." % user.screenname)
user.setState("underway")
d = ftwitter.addUserByScreenname(self.cache, self.endpoint, user)
d.addCallbacks(_added, _failed)
d.addErrback(log.err)
return d
def cancel(self, screenname):
log.msg("Attempting cancel of %r addition." % screenname)
try:
user = self.users[screenname.lower()]
except KeyError:
raise Exception("Cannot cancel unknown user %r." % screenname)
else:
if user.state == "underway":
for item in self.rdq.underway():
if item.job.screenname == screenname:
log.msg("Cancelling underway %r addition." % screenname)
item.cancel()
user.setState("canceled")
break
else:
raise Exception("Could not find %r in underway list." % screenname)
elif user.state == "queued":
queued = self.rdq.pending()
for i, u in enumerate(queued):
if u.screenname == screenname:
del queued[i]
user.setState("canceled")
log.msg("Canceled queued %r addition." % screenname)
break
else:
raise Exception("Could not find %r in queued list." % screenname)
else:
user.setState("canceled")
def added(self, screenname):
try:
user = self.users[screenname.lower()]
except KeyError:
return False
else:
return user.state == "added"
def known(self, screenname):
return screenname.lower() in self.users
def statusSummary(self, screennames):
position = {}
for i, user in enumerate(self.rdq.pending()):
position[user.screenname.lower()] = i
log.msg("position dict is %r" % (position,))
queued = []
underway = []
added = []
canceled = []
failed = []
unknown = []
for screenname in screennames:
try:
user = self.users[screenname.lower()]
except KeyError:
unknown.append(screenname)
else:
log.msg("user: %s" % user)
state = user.state
if state == "queued":
try:
pos = position[screenname.lower()]
except KeyError:
log.msg("ERROR: User %r has no queue position." % screenname)
pos = -1
queued.append([screenname, user.nFriends, pos])
elif state == "underway":
underway.append([screenname, user.nFriends, float(user.workDone) / float(user.workToDo)])
elif state == "added":
added.append(screenname)
elif state == "canceled":
canceled.append(screenname)
elif state == "failed":
failed.append(screenname)
else:
log.msg("ERROR: User %r is in an unknown state: %r" % (screenname, state))
return {
"queued": queued,
"underway": underway,
"added": added, # NB: 'added' is referred to in ftwitter.py
"failed": failed,
"canceled": canceled,
"unknown": unknown,
}
@defer.inlineCallbacks
def close(self):
pending = yield self.rdq.stop()
if pending:
log.msg("Pending user additions canceled: %r" % [p.screenname for p in pending])
super(AdderCache, self).close()
class AdderCache(DumpingCache):
def __init__(self, cache, queueWidth, endpoint):
super(AdderCache, self).__init__()
self.cache = cache
self.queueWidth = queueWidth
self.endpoint = endpoint
def load(self, cacheFile):
self.rdq = ResizableDispatchQueue(self._addUser, width=self.queueWidth)
self.users = super(AdderCache, self).load(cacheFile)
if self.users is None:
self.users = {}
self.setCache(self.users)
else:
added = [u for u in self.users.values() if u.state == 'added']
notAdded = [u for u in self.users.values() if u.state != 'added']
log.msg('Loaded adder cache: found %d added, %d unadded users' %
(len(added), len(notAdded)))
if self.cache.restoreAddQueue:
log.msg('Restoring add queue.')
# Re-queue, in original queue order, any users that are
# marked as having been added.
for user in sorted(notAdded, key=attrgetter('queuedAt')):
log.msg('Restoring %r (previous state %r)' %
(user.screenname, user.state))
user.reset()
d = self.rdq.put(user)
d.addErrback(self._reportCancelled, user.screenname)
self.clean = False
else:
log.msg('Not restoring formerly queued names.')
# Drop users that were not added last time.
for user in notAdded:
log.msg('Dropping user %r (in state %r)' %
(user.screenname, user.state))
del self.users[user.screenname.lower()]
self.clean = False
def __str__(self):
s = ['%d users in adder cache' % len(self.users)]
for key in sorted(self.users.keys()):
s.append(str(self.users[key]))
return '\n'.join(s)
def put(self, screenname, nFriends):
screennameLower = screenname.lower()
user = self.users.get(screennameLower)
if user:
user.nFriends = nFriends
user.setState('queued')
else:
user = User(screenname, nFriends)
self.users[screennameLower] = user
log.msg('Adding screenname %r to request queue.' % screenname)
self.clean = False
d = self.rdq.put(user)
d.addErrback(self._reportCancelled, screenname)
def _addUser(self, user):
def _added(result):
user.setState('added')
self.clean = False
return result
def _failed(fail):
self.clean = False
if fail.check(ftwitter.Canceled):
# The state has been changed to canceled below.
assert user.canceled()
log.msg('Addition of user %r canceled.' % user.screenname)
else:
user.setState('failed')
log.msg('Failed to add %r: %s' % (user.screenname, fail))
log.msg('User %r received from request queue.' % user.screenname)
user.setState('underway')
d = ftwitter.addUserByScreenname(self.cache, self.endpoint, user)
d.addCallbacks(_added, _failed)
d.addErrback(log.err)
return d
def cancel(self, screenname):
log.msg('Attempting cancel of %r addition.' % screenname)
try:
user = self.users[screenname.lower()]
except KeyError:
raise Exception('Cannot cancel unknown user %r.' % screenname)
else:
if user.state == 'underway' or user.state == 'queued':
for job in self.rdq.underway() + self.rdq.pending():
u = job.jobarg
if u.screenname == screenname:
log.msg('Cancelling %s %r addition.' %
(user.state, screenname))
job.cancel()
user.setState('canceled')
break
else:
raise Exception('Could not find %r in underway '
'or pending lists.' % screenname)
else:
user.setState('canceled')
def added(self, screenname):
try:
user = self.users[screenname.lower()]
except KeyError:
return False
else:
return user.state == 'added'
def known(self, screenname):
return screenname.lower() in self.users
def statusSummary(self, screennames):
position = {}
for i, user in enumerate(
[job.jobarg for job in self.rdq.pending()]):
position[user.screenname.lower()] = i
queued = []
underway = []
added = []
canceled = []
failed = []
unknown = []
for screenname in screennames:
try:
user = self.users[screenname.lower()]
except KeyError:
unknown.append(screenname)
else:
log.msg('user: %s' % user)
state = user.state
if state == 'queued':
try:
pos = position[screenname.lower()]
except KeyError:
log.msg('ERROR: User %r has no queue position.' %
screenname)
pos = -1
queued.append([screenname, user.nFriends, pos])
elif state == 'underway':
underway.append(
[screenname, user.nFriends,
float(user.workDone) / float(user.workToDo)])
elif state == 'added':
added.append(screenname)
elif state == 'canceled':
canceled.append(screenname)
elif state == 'failed':
failed.append(screenname)
else:
log.msg('ERROR: User %r is in an unknown state: %r' %
(screenname, state))
return {
'queued': queued,
'underway': underway,
'added': added, # NB: 'added' is referred to in ftwitter.py
'failed': failed,
'canceled': canceled,
'unknown': unknown,
}
@defer.inlineCallbacks
def close(self):
pending = yield self.rdq.stop()
if pending:
log.msg('Pending user additions canceled: %r' %
[p.screenname for p in pending])
super(AdderCache, self).close()
def _reportCancelled(self, fail, screenname):
"""
A user addition was cancelled. Log it and absorb the failure.
"""
fail.trap(defer.CancelledError)
log.msg('Addition of user %r cancelled.' % screenname)
