def __init__(self, statedir):
Thread.__init__(self, name="HostCache")
self._ip2i = IPResolver(cachedir=statedir, maxtime=15)
self._cv = Condition()
self._stopme = False
self._requests = []
self._last_purge = time.time()
self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS))
cherrypy.engine.subscribe('start', self.start)
cherrypy.engine.subscribe('stop', self.stop, priority=100)
def __init__(self, statedir):
Thread.__init__(self, name = "HostCache")
self._ip2i = IPResolver(cachedir = statedir, maxtime=15)
self._cv = Condition()
self._stopme = False
self._requests = []
self._last_purge = time.time()
self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS))
cherrypy.engine.subscribe('start', self.start)
cherrypy.engine.subscribe('stop', self.stop, priority=100)
class HostCache(Thread):
"""Utility to resolve host information."""
_PURGE_INTERVAL = 4 * 3600
_NUM_SIGS = 8
def __init__(self, statedir):
Thread.__init__(self, name="HostCache")
self._ip2i = IPResolver(cachedir=statedir, maxtime=15)
self._cv = Condition()
self._stopme = False
self._requests = []
self._last_purge = time.time()
self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS))
cherrypy.engine.subscribe('start', self.start)
cherrypy.engine.subscribe('stop', self.stop, priority=100)
def _purge(self):
now = time.time()
debug("HOSTDATA", 1, "purging address resolver")
self._last_purge = time.time()
self._ip2i.purge()
def statistics(self):
with self._cv:
return self._ip2i.statistics()
def reset_statistics(self):
with self._cv:
self._ip2i.reset_statistics()
def purge(self):
with self._cv:
self._purge()
def stop(self):
debug("HOSTDATA", 1, "requesting to stop resolved thread")
with self._cv:
self._stopme = True
self._cv.notifyAll()
def lookup(self, kind, hosts, maxwait=30):
"""
Lookup information either by IP address or host name.
:arg str kind: "ip" or "name"
:arg list hosts: list of host name string, ip address or a real name
:arg float maxwait: maximum time in seconds to wait for a result.
"""
reply = Reply()
reply.kind = kind
reply.until = time.time() + maxwait
reply.signal = random.choice(self._signals)
reply.pending = set(hosts)
with self._cv:
self._requests.append(Task(kind, hosts, reply))
self._cv.notifyAll()
with reply.signal:
while True:
if self._stopme:
raise RuntimeError("server stopped")
elif reply.error:
raise reply.error
elif not reply.pending:
reply.finished = True
return reply.result
else:
reply.signal.wait()
def run(self):
with self._cv:
while not self._stopme:
npending = 0
ncurreq = len(self._requests)
# Insert any new requests. If they fail, remember the error.
for r in self._requests:
if not r.reply.submitted:
debug("HOSTDATA", 1, "submitting request: %s %s",
r.kind, r.hosts)
r.reply.submitted = True
try:
self._ip2i.submit(r.hosts,
kind=r.kind,
callback=r.reply)
except Exception, e:
r.reply.error = e
# Pump any pending lookups for up to .25 seconds. Note that this
# will wait only as long as needed, and will quit immediately
# if there is no work at all. It's not unusual we need to wait
# longer than this for final results; see the check further on.
try:
self._cv.release()
npending = self._ip2i.process(.25)
finally:
self._cv.acquire()
# Post-process requests. Remove fully completed, expired and
# failed lookups from the request queue.
nmodified = 0
now = time.time()
for r in self._requests[:]:
rr = r.reply
if rr.finished:
debug("HOSTDATA", 2, "request completed: %s %s",
r.kind, r.hosts)
self._requests.remove(r)
nmodified += 1
elif rr.submitted and rr.until < now:
debug("HOSTDATA", 1, "request has expired: %s %s",
r.kind, r.hosts)
self._requests.remove(r)
with rr.signal:
rr.error = RuntimeError(
"maximum wait time exhausted")
rr.signal.notifyAll()
nmodified += 1
elif rr.submitted and rr.error:
debug("HOSTDATA", 1, "request failed: %s %s", r.kind,
r.hosts)
self._requests.remove(r)
with rr.signal:
rr.signal.notifyAll()
nmodified += 1
# Wait to be notified, but only if we don't already have work to do.
skipwait = (self._stopme or npending or nmodified
or len(self._requests) != ncurreq)
debug("HOSTDATA", 2,
("wait for signal, %d pending, %d requests"
" now vs. %d before, %d modified: %s"), npending,
len(self._requests), ncurreq, nmodified,
(skipwait and "skipping unnecessary wait") or "waiting")
if not skipwait:
if now - self._last_purge > self._PURGE_INTERVAL:
self._purge()
self._cv.wait((self._requests and 0.25) or None)
debug("HOSTDATA", 2, "wait done")
debug("HOSTDATA", 1, "server thread stopped")
class HostCache(Thread):
"""Utility to resolve host information."""
_PURGE_INTERVAL = 4*3600
_NUM_SIGS = 8
def __init__(self, statedir):
Thread.__init__(self, name = "HostCache")
self._ip2i = IPResolver(cachedir = statedir, maxtime=15)
self._cv = Condition()
self._stopme = False
self._requests = []
self._last_purge = time.time()
self._signals = map(lambda x: Condition(), xrange(0, self._NUM_SIGS))
cherrypy.engine.subscribe('start', self.start)
cherrypy.engine.subscribe('stop', self.stop, priority=100)
def _purge(self):
now = time.time()
debug("HOSTDATA", 1, "purging address resolver")
self._last_purge = time.time()
self._ip2i.purge()
def statistics(self):
with self._cv:
return self._ip2i.statistics()
def reset_statistics(self):
with self._cv:
self._ip2i.reset_statistics()
def purge(self):
with self._cv:
self._purge()
def stop(self):
debug("HOSTDATA", 1, "requesting to stop resolved thread")
with self._cv:
self._stopme = True
self._cv.notifyAll()
def lookup(self, kind, hosts, maxwait=30):
"""
Lookup information either by IP address or host name.
:arg str kind: "ip" or "name"
:arg list hosts: list of host name string, ip address or a real name
:arg float maxwait: maximum time in seconds to wait for a result.
"""
reply = Reply()
reply.kind = kind
reply.until = time.time() + maxwait
reply.signal = random.choice(self._signals)
reply.pending = set(hosts)
with self._cv:
self._requests.append(Task(kind, hosts, reply))
self._cv.notifyAll()
with reply.signal:
while True:
if self._stopme:
raise RuntimeError("server stopped")
elif reply.error:
raise reply.error
elif not reply.pending:
reply.finished = True
return reply.result
else:
reply.signal.wait()
def run(self):
with self._cv:
while not self._stopme:
npending = 0
ncurreq = len(self._requests)
# Insert any new requests. If they fail, remember the error.
for r in self._requests:
if not r.reply.submitted:
debug("HOSTDATA", 1, "submitting request: %s %s", r.kind, r.hosts)
r.reply.submitted = True
try:
self._ip2i.submit(r.hosts, kind=r.kind, callback=r.reply)
except Exception, e:
r.reply.error = e
# Pump any pending lookups for up to .25 seconds. Note that this
# will wait only as long as needed, and will quit immediately
# if there is no work at all. It's not unusual we need to wait
# longer than this for final results; see the check further on.
try:
self._cv.release()
npending = self._ip2i.process(.25)
finally:
self._cv.acquire()
# Post-process requests. Remove fully completed, expired and
# failed lookups from the request queue.
nmodified = 0
now = time.time()
for r in self._requests[:]:
rr = r.reply
if rr.finished:
debug("HOSTDATA", 2, "request completed: %s %s", r.kind, r.hosts)
self._requests.remove(r)
nmodified += 1
elif rr.submitted and rr.until < now:
debug("HOSTDATA", 1, "request has expired: %s %s", r.kind, r.hosts)
self._requests.remove(r)
with rr.signal:
rr.error = RuntimeError("maximum wait time exhausted")
rr.signal.notifyAll()
nmodified += 1
elif rr.submitted and rr.error:
debug("HOSTDATA", 1, "request failed: %s %s", r.kind, r.hosts)
self._requests.remove(r)
with rr.signal:
rr.signal.notifyAll()
nmodified += 1
# Wait to be notified, but only if we don't already have work to do.
skipwait = (self._stopme or npending or nmodified
or len(self._requests) != ncurreq)
debug("HOSTDATA", 2, ("wait for signal, %d pending, %d requests"
" now vs. %d before, %d modified: %s"),
npending, len(self._requests), ncurreq, nmodified,
(skipwait and "skipping unnecessary wait") or "waiting")
if not skipwait:
if now - self._last_purge > self._PURGE_INTERVAL:
self._purge()
self._cv.wait((self._requests and 0.25) or None)
debug("HOSTDATA", 2, "wait done")
debug("HOSTDATA", 1, "server thread stopped")
