def __init__(self, statedir): Thread.__init__(self, name="HostCache") self._ip2i = IPResolver(cachedir=statedir, maxtime=15) self._cv = Condition() self._stopme = False self._requests = [] self._last_purge = time.time() self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS)) cherrypy.engine.subscribe('start', self.start) cherrypy.engine.subscribe('stop', self.stop, priority=100)
def __init__(self, statedir): Thread.__init__(self, name = "HostCache") self._ip2i = IPResolver(cachedir = statedir, maxtime=15) self._cv = Condition() self._stopme = False self._requests = [] self._last_purge = time.time() self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS)) cherrypy.engine.subscribe('start', self.start) cherrypy.engine.subscribe('stop', self.stop, priority=100)
class HostCache(Thread): """Utility to resolve host information.""" _PURGE_INTERVAL = 4 * 3600 _NUM_SIGS = 8 def __init__(self, statedir): Thread.__init__(self, name="HostCache") self._ip2i = IPResolver(cachedir=statedir, maxtime=15) self._cv = Condition() self._stopme = False self._requests = [] self._last_purge = time.time() self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS)) cherrypy.engine.subscribe('start', self.start) cherrypy.engine.subscribe('stop', self.stop, priority=100) def _purge(self): now = time.time() debug("HOSTDATA", 1, "purging address resolver") self._last_purge = time.time() self._ip2i.purge() def statistics(self): with self._cv: return self._ip2i.statistics() def reset_statistics(self): with self._cv: self._ip2i.reset_statistics() def purge(self): with self._cv: self._purge() def stop(self): debug("HOSTDATA", 1, "requesting to stop resolved thread") with self._cv: self._stopme = True self._cv.notifyAll() def lookup(self, kind, hosts, maxwait=30): """ Lookup information either by IP address or host name. :arg str kind: "ip" or "name" :arg list hosts: list of host name string, ip address or a real name :arg float maxwait: maximum time in seconds to wait for a result. """ reply = Reply() reply.kind = kind reply.until = time.time() + maxwait reply.signal = random.choice(self._signals) reply.pending = set(hosts) with self._cv: self._requests.append(Task(kind, hosts, reply)) self._cv.notifyAll() with reply.signal: while True: if self._stopme: raise RuntimeError("server stopped") elif reply.error: raise reply.error elif not reply.pending: reply.finished = True return reply.result else: reply.signal.wait() def run(self): with self._cv: while not self._stopme: npending = 0 ncurreq = len(self._requests) # Insert any new requests. If they fail, remember the error. for r in self._requests: if not r.reply.submitted: debug("HOSTDATA", 1, "submitting request: %s %s", r.kind, r.hosts) r.reply.submitted = True try: self._ip2i.submit(r.hosts, kind=r.kind, callback=r.reply) except Exception, e: r.reply.error = e # Pump any pending lookups for up to .25 seconds. Note that this # will wait only as long as needed, and will quit immediately # if there is no work at all. It's not unusual we need to wait # longer than this for final results; see the check further on. try: self._cv.release() npending = self._ip2i.process(.25) finally: self._cv.acquire() # Post-process requests. Remove fully completed, expired and # failed lookups from the request queue. nmodified = 0 now = time.time() for r in self._requests[:]: rr = r.reply if rr.finished: debug("HOSTDATA", 2, "request completed: %s %s", r.kind, r.hosts) self._requests.remove(r) nmodified += 1 elif rr.submitted and rr.until < now: debug("HOSTDATA", 1, "request has expired: %s %s", r.kind, r.hosts) self._requests.remove(r) with rr.signal: rr.error = RuntimeError( "maximum wait time exhausted") rr.signal.notifyAll() nmodified += 1 elif rr.submitted and rr.error: debug("HOSTDATA", 1, "request failed: %s %s", r.kind, r.hosts) self._requests.remove(r) with rr.signal: rr.signal.notifyAll() nmodified += 1 # Wait to be notified, but only if we don't already have work to do. skipwait = (self._stopme or npending or nmodified or len(self._requests) != ncurreq) debug("HOSTDATA", 2, ("wait for signal, %d pending, %d requests" " now vs. %d before, %d modified: %s"), npending, len(self._requests), ncurreq, nmodified, (skipwait and "skipping unnecessary wait") or "waiting") if not skipwait: if now - self._last_purge > self._PURGE_INTERVAL: self._purge() self._cv.wait((self._requests and 0.25) or None) debug("HOSTDATA", 2, "wait done") debug("HOSTDATA", 1, "server thread stopped")
class HostCache(Thread): """Utility to resolve host information.""" _PURGE_INTERVAL = 4*3600 _NUM_SIGS = 8 def __init__(self, statedir): Thread.__init__(self, name = "HostCache") self._ip2i = IPResolver(cachedir = statedir, maxtime=15) self._cv = Condition() self._stopme = False self._requests = [] self._last_purge = time.time() self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS)) cherrypy.engine.subscribe('start', self.start) cherrypy.engine.subscribe('stop', self.stop, priority=100) def _purge(self): now = time.time() debug("HOSTDATA", 1, "purging address resolver") self._last_purge = time.time() self._ip2i.purge() def statistics(self): with self._cv: return self._ip2i.statistics() def reset_statistics(self): with self._cv: self._ip2i.reset_statistics() def purge(self): with self._cv: self._purge() def stop(self): debug("HOSTDATA", 1, "requesting to stop resolved thread") with self._cv: self._stopme = True self._cv.notifyAll() def lookup(self, kind, hosts, maxwait=30): """ Lookup information either by IP address or host name. :arg str kind: "ip" or "name" :arg list hosts: list of host name string, ip address or a real name :arg float maxwait: maximum time in seconds to wait for a result. """ reply = Reply() reply.kind = kind reply.until = time.time() + maxwait reply.signal = random.choice(self._signals) reply.pending = set(hosts) with self._cv: self._requests.append(Task(kind, hosts, reply)) self._cv.notifyAll() with reply.signal: while True: if self._stopme: raise RuntimeError("server stopped") elif reply.error: raise reply.error elif not reply.pending: reply.finished = True return reply.result else: reply.signal.wait() def run(self): with self._cv: while not self._stopme: npending = 0 ncurreq = len(self._requests) # Insert any new requests. If they fail, remember the error. for r in self._requests: if not r.reply.submitted: debug("HOSTDATA", 1, "submitting request: %s %s", r.kind, r.hosts) r.reply.submitted = True try: self._ip2i.submit(r.hosts, kind=r.kind, callback=r.reply) except Exception, e: r.reply.error = e # Pump any pending lookups for up to .25 seconds. Note that this # will wait only as long as needed, and will quit immediately # if there is no work at all. It's not unusual we need to wait # longer than this for final results; see the check further on. try: self._cv.release() npending = self._ip2i.process(.25) finally: self._cv.acquire() # Post-process requests. Remove fully completed, expired and # failed lookups from the request queue. nmodified = 0 now = time.time() for r in self._requests[:]: rr = r.reply if rr.finished: debug("HOSTDATA", 2, "request completed: %s %s", r.kind, r.hosts) self._requests.remove(r) nmodified += 1 elif rr.submitted and rr.until < now: debug("HOSTDATA", 1, "request has expired: %s %s", r.kind, r.hosts) self._requests.remove(r) with rr.signal: rr.error = RuntimeError("maximum wait time exhausted") rr.signal.notifyAll() nmodified += 1 elif rr.submitted and rr.error: debug("HOSTDATA", 1, "request failed: %s %s", r.kind, r.hosts) self._requests.remove(r) with rr.signal: rr.signal.notifyAll() nmodified += 1 # Wait to be notified, but only if we don't already have work to do. skipwait = (self._stopme or npending or nmodified or len(self._requests) != ncurreq) debug("HOSTDATA", 2, ("wait for signal, %d pending, %d requests" " now vs. %d before, %d modified: %s"), npending, len(self._requests), ncurreq, nmodified, (skipwait and "skipping unnecessary wait") or "waiting") if not skipwait: if now - self._last_purge > self._PURGE_INTERVAL: self._purge() self._cv.wait((self._requests and 0.25) or None) debug("HOSTDATA", 2, "wait done") debug("HOSTDATA", 1, "server thread stopped")