def testUnstarted(self): """ Check basic facts about a fresh instance of a ResizableDispatchQueue. """ dq = ResizableDispatchQueue(None) self.assertEqual(dq.pending(), []) self.assertEqual(dq.underway(), set()) self.assertFalse(dq.stopped) self.assertFalse(dq.paused) self.assertEqual((0, 0), dq.size()) return dq.stop()
class AdderCache(DumpingCache): def __init__(self, cache, queueWidth, endpoint): super(AdderCache, self).__init__() self.cache = cache self.queueWidth = queueWidth self.endpoint = endpoint def load(self, cacheFile): self.rdq = ResizableDispatchQueue(self._addUser, width=self.queueWidth) self.users = super(AdderCache, self).load(cacheFile) if self.users is None: self.users = {} self.setCache(self.users) else: added = [u for u in self.users.values() if u.state == "added"] notAdded = [u for u in self.users.values() if u.state != "added"] log.msg("Loaded adder cache: found %d added, %d unadded users" % (len(added), len(notAdded))) if self.cache.restoreAddQueue: log.msg("Restoring add queue.") # Re-queue, in original queue order, any users that are # marked as having been added. for user in sorted(notAdded, key=attrgetter("queuedAt")): log.msg("Restoring %r (previous state %r)" % (user.screenname, user.state)) user.reset() self.rdq.put(user) self.clean = False else: log.msg("Not restoring formerly queued names.") # Drop users that were not added last time. for user in notAdded: log.msg("Dropping user %r (in state %r)" % (user.screenname, user.state)) del self.users[user.screenname.lower()] self.clean = False def __str__(self): s = ["%d users in adder cache" % len(self.users)] for key in sorted(self.users.keys()): s.append(str(self.users[key])) return "\n".join(s) def put(self, screenname, nFriends): screennameLower = screenname.lower() user = self.users.get(screennameLower) if user: user.nFriends = nFriends user.setState("queued") else: user = User(screenname, nFriends) self.users[screennameLower] = user log.msg("Adding screenname %r to request queue." % screenname) self.clean = False self.rdq.put(user) def _addUser(self, user): def _added(result): user.setState("added") self.clean = False return result def _failed(fail): self.clean = False if fail.check(ftwitter.Canceled): # The state has been changed to canceled below. assert user.canceled() log.msg("Addition of user %r canceled." % user.screenname) else: user.setState("failed") log.msg("Failed to add %r: %s" % (user.screenname, fail)) log.msg("User %r received from request queue." % user.screenname) user.setState("underway") d = ftwitter.addUserByScreenname(self.cache, self.endpoint, user) d.addCallbacks(_added, _failed) d.addErrback(log.err) return d def cancel(self, screenname): log.msg("Attempting cancel of %r addition." % screenname) try: user = self.users[screenname.lower()] except KeyError: raise Exception("Cannot cancel unknown user %r." % screenname) else: if user.state == "underway": for item in self.rdq.underway(): if item.job.screenname == screenname: log.msg("Cancelling underway %r addition." % screenname) item.cancel() user.setState("canceled") break else: raise Exception("Could not find %r in underway list." % screenname) elif user.state == "queued": queued = self.rdq.pending() for i, u in enumerate(queued): if u.screenname == screenname: del queued[i] user.setState("canceled") log.msg("Canceled queued %r addition." % screenname) break else: raise Exception("Could not find %r in queued list." % screenname) else: user.setState("canceled") def added(self, screenname): try: user = self.users[screenname.lower()] except KeyError: return False else: return user.state == "added" def known(self, screenname): return screenname.lower() in self.users def statusSummary(self, screennames): position = {} for i, user in enumerate(self.rdq.pending()): position[user.screenname.lower()] = i log.msg("position dict is %r" % (position,)) queued = [] underway = [] added = [] canceled = [] failed = [] unknown = [] for screenname in screennames: try: user = self.users[screenname.lower()] except KeyError: unknown.append(screenname) else: log.msg("user: %s" % user) state = user.state if state == "queued": try: pos = position[screenname.lower()] except KeyError: log.msg("ERROR: User %r has no queue position." % screenname) pos = -1 queued.append([screenname, user.nFriends, pos]) elif state == "underway": underway.append([screenname, user.nFriends, float(user.workDone) / float(user.workToDo)]) elif state == "added": added.append(screenname) elif state == "canceled": canceled.append(screenname) elif state == "failed": failed.append(screenname) else: log.msg("ERROR: User %r is in an unknown state: %r" % (screenname, state)) return { "queued": queued, "underway": underway, "added": added, # NB: 'added' is referred to in ftwitter.py "failed": failed, "canceled": canceled, "unknown": unknown, } @defer.inlineCallbacks def close(self): pending = yield self.rdq.stop() if pending: log.msg("Pending user additions canceled: %r" % [p.screenname for p in pending]) super(AdderCache, self).close()
class AdderCache(DumpingCache): def __init__(self, cache, queueWidth, endpoint): super(AdderCache, self).__init__() self.cache = cache self.queueWidth = queueWidth self.endpoint = endpoint def load(self, cacheFile): self.rdq = ResizableDispatchQueue(self._addUser, width=self.queueWidth) self.users = super(AdderCache, self).load(cacheFile) if self.users is None: self.users = {} self.setCache(self.users) else: added = [u for u in self.users.values() if u.state == 'added'] notAdded = [u for u in self.users.values() if u.state != 'added'] log.msg('Loaded adder cache: found %d added, %d unadded users' % (len(added), len(notAdded))) if self.cache.restoreAddQueue: log.msg('Restoring add queue.') # Re-queue, in original queue order, any users that are # marked as having been added. for user in sorted(notAdded, key=attrgetter('queuedAt')): log.msg('Restoring %r (previous state %r)' % (user.screenname, user.state)) user.reset() d = self.rdq.put(user) d.addErrback(self._reportCancelled, user.screenname) self.clean = False else: log.msg('Not restoring formerly queued names.') # Drop users that were not added last time. for user in notAdded: log.msg('Dropping user %r (in state %r)' % (user.screenname, user.state)) del self.users[user.screenname.lower()] self.clean = False def __str__(self): s = ['%d users in adder cache' % len(self.users)] for key in sorted(self.users.keys()): s.append(str(self.users[key])) return '\n'.join(s) def put(self, screenname, nFriends): screennameLower = screenname.lower() user = self.users.get(screennameLower) if user: user.nFriends = nFriends user.setState('queued') else: user = User(screenname, nFriends) self.users[screennameLower] = user log.msg('Adding screenname %r to request queue.' % screenname) self.clean = False d = self.rdq.put(user) d.addErrback(self._reportCancelled, screenname) def _addUser(self, user): def _added(result): user.setState('added') self.clean = False return result def _failed(fail): self.clean = False if fail.check(ftwitter.Canceled): # The state has been changed to canceled below. assert user.canceled() log.msg('Addition of user %r canceled.' % user.screenname) else: user.setState('failed') log.msg('Failed to add %r: %s' % (user.screenname, fail)) log.msg('User %r received from request queue.' % user.screenname) user.setState('underway') d = ftwitter.addUserByScreenname(self.cache, self.endpoint, user) d.addCallbacks(_added, _failed) d.addErrback(log.err) return d def cancel(self, screenname): log.msg('Attempting cancel of %r addition.' % screenname) try: user = self.users[screenname.lower()] except KeyError: raise Exception('Cannot cancel unknown user %r.' % screenname) else: if user.state == 'underway' or user.state == 'queued': for job in self.rdq.underway() + self.rdq.pending(): u = job.jobarg if u.screenname == screenname: log.msg('Cancelling %s %r addition.' % (user.state, screenname)) job.cancel() user.setState('canceled') break else: raise Exception('Could not find %r in underway ' 'or pending lists.' % screenname) else: user.setState('canceled') def added(self, screenname): try: user = self.users[screenname.lower()] except KeyError: return False else: return user.state == 'added' def known(self, screenname): return screenname.lower() in self.users def statusSummary(self, screennames): position = {} for i, user in enumerate( [job.jobarg for job in self.rdq.pending()]): position[user.screenname.lower()] = i queued = [] underway = [] added = [] canceled = [] failed = [] unknown = [] for screenname in screennames: try: user = self.users[screenname.lower()] except KeyError: unknown.append(screenname) else: log.msg('user: %s' % user) state = user.state if state == 'queued': try: pos = position[screenname.lower()] except KeyError: log.msg('ERROR: User %r has no queue position.' % screenname) pos = -1 queued.append([screenname, user.nFriends, pos]) elif state == 'underway': underway.append( [screenname, user.nFriends, float(user.workDone) / float(user.workToDo)]) elif state == 'added': added.append(screenname) elif state == 'canceled': canceled.append(screenname) elif state == 'failed': failed.append(screenname) else: log.msg('ERROR: User %r is in an unknown state: %r' % (screenname, state)) return { 'queued': queued, 'underway': underway, 'added': added, # NB: 'added' is referred to in ftwitter.py 'failed': failed, 'canceled': canceled, 'unknown': unknown, } @defer.inlineCallbacks def close(self): pending = yield self.rdq.stop() if pending: log.msg('Pending user additions canceled: %r' % [p.screenname for p in pending]) super(AdderCache, self).close() def _reportCancelled(self, fail, screenname): """ A user addition was cancelled. Log it and absorb the failure. """ fail.trap(defer.CancelledError) log.msg('Addition of user %r cancelled.' % screenname)