def testUnstarted(self):
"""
Check basic facts about a fresh instance of a
ResizableDispatchQueue.
"""
dq = ResizableDispatchQueue(None)
self.assertEqual(dq.pending(), [])
self.assertEqual(dq.underway(), set())
self.assertFalse(dq.stopped)
self.assertFalse(dq.paused)
self.assertEqual((0, 0), dq.size())
return dq.stop()
class AdderCache(DumpingCache):
def __init__(self, cache, queueWidth, endpoint):
super(AdderCache, self).__init__()
self.cache = cache
self.queueWidth = queueWidth
self.endpoint = endpoint
def load(self, cacheFile):
self.rdq = ResizableDispatchQueue(self._addUser, width=self.queueWidth)
self.users = super(AdderCache, self).load(cacheFile)
if self.users is None:
self.users = {}
self.setCache(self.users)
else:
added = [u for u in self.users.values() if u.state == "added"]
notAdded = [u for u in self.users.values() if u.state != "added"]
log.msg("Loaded adder cache: found %d added, %d unadded users" % (len(added), len(notAdded)))
if self.cache.restoreAddQueue:
log.msg("Restoring add queue.")
# Re-queue, in original queue order, any users that are
# marked as having been added.
for user in sorted(notAdded, key=attrgetter("queuedAt")):
log.msg("Restoring %r (previous state %r)" % (user.screenname, user.state))
user.reset()
self.rdq.put(user)
self.clean = False
else:
log.msg("Not restoring formerly queued names.")
# Drop users that were not added last time.
for user in notAdded:
log.msg("Dropping user %r (in state %r)" % (user.screenname, user.state))
del self.users[user.screenname.lower()]
self.clean = False
def __str__(self):
s = ["%d users in adder cache" % len(self.users)]
for key in sorted(self.users.keys()):
s.append(str(self.users[key]))
return "\n".join(s)
def put(self, screenname, nFriends):
screennameLower = screenname.lower()
user = self.users.get(screennameLower)
if user:
user.nFriends = nFriends
user.setState("queued")
else:
user = User(screenname, nFriends)
self.users[screennameLower] = user
log.msg("Adding screenname %r to request queue." % screenname)
self.clean = False
self.rdq.put(user)
def _addUser(self, user):
def _added(result):
user.setState("added")
self.clean = False
return result
def _failed(fail):
self.clean = False
if fail.check(ftwitter.Canceled):
# The state has been changed to canceled below.
assert user.canceled()
log.msg("Addition of user %r canceled." % user.screenname)
else:
user.setState("failed")
log.msg("Failed to add %r: %s" % (user.screenname, fail))
log.msg("User %r received from request queue." % user.screenname)
user.setState("underway")
d = ftwitter.addUserByScreenname(self.cache, self.endpoint, user)
d.addCallbacks(_added, _failed)
d.addErrback(log.err)
return d
def cancel(self, screenname):
log.msg("Attempting cancel of %r addition." % screenname)
try:
user = self.users[screenname.lower()]
except KeyError:
raise Exception("Cannot cancel unknown user %r." % screenname)
else:
if user.state == "underway":
for item in self.rdq.underway():
if item.job.screenname == screenname:
log.msg("Cancelling underway %r addition." % screenname)
item.cancel()
user.setState("canceled")
break
else:
raise Exception("Could not find %r in underway list." % screenname)
elif user.state == "queued":
queued = self.rdq.pending()
for i, u in enumerate(queued):
if u.screenname == screenname:
del queued[i]
user.setState("canceled")
log.msg("Canceled queued %r addition." % screenname)
break
else:
raise Exception("Could not find %r in queued list." % screenname)
else:
user.setState("canceled")
def added(self, screenname):
try:
user = self.users[screenname.lower()]
except KeyError:
return False
else:
return user.state == "added"
def known(self, screenname):
return screenname.lower() in self.users
def statusSummary(self, screennames):
position = {}
for i, user in enumerate(self.rdq.pending()):
position[user.screenname.lower()] = i
log.msg("position dict is %r" % (position,))
queued = []
underway = []
added = []
canceled = []
failed = []
unknown = []
for screenname in screennames:
try:
user = self.users[screenname.lower()]
except KeyError:
unknown.append(screenname)
else:
log.msg("user: %s" % user)
state = user.state
if state == "queued":
try:
pos = position[screenname.lower()]
except KeyError:
log.msg("ERROR: User %r has no queue position." % screenname)
pos = -1
queued.append([screenname, user.nFriends, pos])
elif state == "underway":
underway.append([screenname, user.nFriends, float(user.workDone) / float(user.workToDo)])
elif state == "added":
added.append(screenname)
elif state == "canceled":
canceled.append(screenname)
elif state == "failed":
failed.append(screenname)
else:
log.msg("ERROR: User %r is in an unknown state: %r" % (screenname, state))
return {
"queued": queued,
"underway": underway,
"added": added, # NB: 'added' is referred to in ftwitter.py
"failed": failed,
"canceled": canceled,
"unknown": unknown,
}
@defer.inlineCallbacks
def close(self):
pending = yield self.rdq.stop()
if pending:
log.msg("Pending user additions canceled: %r" % [p.screenname for p in pending])
super(AdderCache, self).close()
class AdderCache(DumpingCache):
def __init__(self, cache, queueWidth, endpoint):
super(AdderCache, self).__init__()
self.cache = cache
self.queueWidth = queueWidth
self.endpoint = endpoint
def load(self, cacheFile):
self.rdq = ResizableDispatchQueue(self._addUser, width=self.queueWidth)
self.users = super(AdderCache, self).load(cacheFile)
if self.users is None:
self.users = {}
self.setCache(self.users)
else:
added = [u for u in self.users.values() if u.state == 'added']
notAdded = [u for u in self.users.values() if u.state != 'added']
log.msg('Loaded adder cache: found %d added, %d unadded users' %
(len(added), len(notAdded)))
if self.cache.restoreAddQueue:
log.msg('Restoring add queue.')
# Re-queue, in original queue order, any users that are
# marked as having been added.
for user in sorted(notAdded, key=attrgetter('queuedAt')):
log.msg('Restoring %r (previous state %r)' %
(user.screenname, user.state))
user.reset()
d = self.rdq.put(user)
d.addErrback(self._reportCancelled, user.screenname)
self.clean = False
else:
log.msg('Not restoring formerly queued names.')
# Drop users that were not added last time.
for user in notAdded:
log.msg('Dropping user %r (in state %r)' %
(user.screenname, user.state))
del self.users[user.screenname.lower()]
self.clean = False
def __str__(self):
s = ['%d users in adder cache' % len(self.users)]
for key in sorted(self.users.keys()):
s.append(str(self.users[key]))
return '\n'.join(s)
def put(self, screenname, nFriends):
screennameLower = screenname.lower()
user = self.users.get(screennameLower)
if user:
user.nFriends = nFriends
user.setState('queued')
else:
user = User(screenname, nFriends)
self.users[screennameLower] = user
log.msg('Adding screenname %r to request queue.' % screenname)
self.clean = False
d = self.rdq.put(user)
d.addErrback(self._reportCancelled, screenname)
def _addUser(self, user):
def _added(result):
user.setState('added')
self.clean = False
return result
def _failed(fail):
self.clean = False
if fail.check(ftwitter.Canceled):
# The state has been changed to canceled below.
assert user.canceled()
log.msg('Addition of user %r canceled.' % user.screenname)
else:
user.setState('failed')
log.msg('Failed to add %r: %s' % (user.screenname, fail))
log.msg('User %r received from request queue.' % user.screenname)
user.setState('underway')
d = ftwitter.addUserByScreenname(self.cache, self.endpoint, user)
d.addCallbacks(_added, _failed)
d.addErrback(log.err)
return d
def cancel(self, screenname):
log.msg('Attempting cancel of %r addition.' % screenname)
try:
user = self.users[screenname.lower()]
except KeyError:
raise Exception('Cannot cancel unknown user %r.' % screenname)
else:
if user.state == 'underway' or user.state == 'queued':
for job in self.rdq.underway() + self.rdq.pending():
u = job.jobarg
if u.screenname == screenname:
log.msg('Cancelling %s %r addition.' %
(user.state, screenname))
job.cancel()
user.setState('canceled')
break
else:
raise Exception('Could not find %r in underway '
'or pending lists.' % screenname)
else:
user.setState('canceled')
def added(self, screenname):
try:
user = self.users[screenname.lower()]
except KeyError:
return False
else:
return user.state == 'added'
def known(self, screenname):
return screenname.lower() in self.users
def statusSummary(self, screennames):
position = {}
for i, user in enumerate(
[job.jobarg for job in self.rdq.pending()]):
position[user.screenname.lower()] = i
queued = []
underway = []
added = []
canceled = []
failed = []
unknown = []
for screenname in screennames:
try:
user = self.users[screenname.lower()]
except KeyError:
unknown.append(screenname)
else:
log.msg('user: %s' % user)
state = user.state
if state == 'queued':
try:
pos = position[screenname.lower()]
except KeyError:
log.msg('ERROR: User %r has no queue position.' %
screenname)
pos = -1
queued.append([screenname, user.nFriends, pos])
elif state == 'underway':
underway.append(
[screenname, user.nFriends,
float(user.workDone) / float(user.workToDo)])
elif state == 'added':
added.append(screenname)
elif state == 'canceled':
canceled.append(screenname)
elif state == 'failed':
failed.append(screenname)
else:
log.msg('ERROR: User %r is in an unknown state: %r' %
(screenname, state))
return {
'queued': queued,
'underway': underway,
'added': added, # NB: 'added' is referred to in ftwitter.py
'failed': failed,
'canceled': canceled,
'unknown': unknown,
}
@defer.inlineCallbacks
def close(self):
pending = yield self.rdq.stop()
if pending:
log.msg('Pending user additions canceled: %r' %
[p.screenname for p in pending])
super(AdderCache, self).close()
def _reportCancelled(self, fail, screenname):
"""
A user addition was cancelled. Log it and absorb the failure.
"""
fail.trap(defer.CancelledError)
log.msg('Addition of user %r cancelled.' % screenname)
