def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout):
'''
Constructor
dsp_endpoints : is a list of tuples(endpoint, qps) where
enpoint is a string like '192.168.10.152:5869'
and qps is the value indicating queries per
second for that enpoint.
balance_conn_timeout : is the time period for rebalancing
available connections.
'''
# list containing tuples in the form
# (endpoint, expected qps, current qps)
self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints]
self.event_endpoint = event_endpoint
self.conns = {}
self.awaiting_conns = {}
self.event_conn_queue = []
self.event_conns = {}
self.event_connections = 0
self.keep_alive_resp_waiting = {}
self.balance_conn_to = balance_conn_timeout
self.loop = pyev.default_loop()
self.watchers = [
pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS
]
self.watchers.append(
pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop,
self.balance))
self.watchers.append(
pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop,
self.check_established_connections))
self.watchers.append(
pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop,
self.check_pending_wins))
if EVENT_CONN_KEEP_ALIVE_TO:
self.watchers.append(
pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO,
self.loop, self.send_keep_alives))
self.current_connections = 0
self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME)
self.adserver = AdServer(self.loop)
self.request_fact.initialize()
self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver)
if PLUGIN_DO_TO:
self.watchers.append(
pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop,
self.request_fact.plugin_instance.do))
self.pending_wins = []
def connect(self):
'''
Connect and set the app layer timeout
'''
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setblocking(0)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
logging.debug('EphemeralConnection : connecting to %s:%d' %
(self.address[0], self.address[1]))
res = self.sock.connect_ex(self.address)
if res != errno.EINPROGRESS:
logging.error(
'EphemeralConnection : unable to connect to %s:%d res=%d' %
(self.address[0], self.address[1], res))
self.state = EphemeralConnection.STATE_ERROR
self.handle_error('unable to connect')
else:
if not self.connect_cb:
self.connect_cb = self.io_cb
self.state = EphemeralConnection.STATE_CONNECTING
self.watcher = pyev.Io(self.sock, pyev.EV_WRITE, self.loop,
self.connect_cb)
self.watcher.start()
# start the application layer time out
self.conn_timer = pyev.Timer(2, 0.0, self.loop, self.too_long_conn,
self)
self.conn_timer.start()
return self.state
def handle_write(self):
try:
logging.debug('EphemeralConnection : handling write %d' % self.id)
self.state = EphemeralConnection.STATE_CONNECTED
logging.debug('EphemeralConnection : sending %s' % self.buf)
sent = self.sock.send(self.buf)
self.timer = pyev.Timer(2, 0.0, self.loop, self.no_response_cb,
self)
self.timer.start()
self.conn_timer.stop()
except socket.error as err:
logging.error('EphemeralConnection : handle_write ex')
if err.args[0] not in NONBLOCKING:
self.handle_error(
"EphemeralConnection : error writing to {0}".format(
self.sock))
else:
logging.error(
'EphemeralConnection : NONBLOCKING event on write')
else:
if self.state == EphemeralConnection.STATE_CONNECTING:
self.state = EphemeralConnection.STATE_CONNECTED
self.buf = self.buf[sent:]
if not self.buf:
# all the request buffer was sent,
# let's wait for the response
self.reset(pyev.EV_READ)
else:
# there is still some buffer left,
# wait for the write event again
logging.info('EphemeralConnection : partial buffer sent')
self.reset(pyev.EV_WRITE)
def _do_wait(self, timeout):
"""Wait for the deferred to be completed for a period of time
Raises TimeoutError if the wait times out before the future is done.
Raises CancelledError if the future is cancelled before the
timeout is done.
"""
if self._cancelled:
raise CancelledError()
if not self._done:
self._wait = True
self._sigint = pyev.Signal(signal.SIGINT, self.loop,
lambda watcher, events: self._cancel(),
None)
self._sigint.start()
if timeout and timeout > 0.0:
self._timer = pyev.Timer(timeout, 0.0, self.loop,
self._clear_wait, None)
self._timer.start()
while self._wait and not self._done and not self._cancelled:
self.loop.start(pyev.EVRUN_ONCE)
if self._cancelled:
raise CancelledError()
elif not self._done:
raise TimeoutError()
def connect_client(self):
client = whizzer.TcpClient(self.loop, self.factory, "127.0.0.1", 2000)
logger.info('client calling connect')
d = client.connect()
logger.info('client called connect')
d.add_callback(self.connect_success)
d.add_errback(self.connect_failed)
self.timer = pyev.Timer(1.0, 0.0, self.loop, self.timeout, None)
self.timer.start()
def later(seconds, cb, *args, **kwargs): def wrap(watcher, events): args, kwargs = watcher.data timed_calls.remove(watcher) try: cb(*args, **kwargs) except: traceback.print_exc() t = pyev.Timer(seconds, 0.0, default_loop, wrap, data=(args, kwargs)) timed_calls.add(t) t.start()
def __init__(self, logger=None):
#self._loop = pyev.default_loop(io_interval=timeout/1000.0, timeout_interval=timeout/1000.0)
#self._loop = pyev.default_loop(timeout_interval=timeout)
self._loop = pyev.default_loop()
self._timer = pyev.Timer(0.01, 0, self._loop, self._timer_callback)
self._timeout = 100
# print self._loop.backend, pyev.EVBACKEND_EPOLL
self._watchers = dict() # key is fd
self._empty = []
self.logger = logger
self._in = pyev.EV_READ
self._out = pyev.EV_WRITE
def __init__(self, loop, classes=[]):
"""Watches the garbage collector and prints out stats
periodically with how many objects are in the gc for a given type.
Useful for debugging situations where referrences are being held
when they shouldn't be. A rather annoying problem in callback
oriented code (that I've personally found anyways).
"""
gc.set_debug(gc.DEBUG_STATS)
self.classes = classes
self.timer = pyev.Timer(5.0, 5.0, loop, self.print_stats)
self.timer.start()
def __init__(self, loop, sock, addr, timeout):
self.loop = loop
self.sock = sock
self.addr = addr
self.timeout = timeout
self.connect_watcher = pyev.Io(self.sock, pyev.EV_WRITE, self.loop,
self._connected)
self.timeout_watcher = pyev.Timer(self.timeout, 0.0, self.loop,
self._timeout)
self.deferred = Deferred(self.loop)
self.started = False
self.connected = False
self.timedout = False
self.errored = False
def __init__(self, config):
logger.debug('hpfeedhandler init')
self.client = hpclient(config['server'], int(config['port']), config['ident'], config['secret'])
ihandler.__init__(self, '*')
self.dynip_resolve = config.get('dynip_resolve', '')
self.dynip_timer = None
self.ownip = None
if self.dynip_resolve and 'http' in self.dynip_resolve:
if pyev == None:
logger.debug('You are missing the python pyev binding in your dionaea installation.')
else:
logger.debug('hpfeedihandler will use dynamic IP resolving!')
self.loop = pyev.default_loop()
self.dynip_timer = pyev.Timer(2., 300, self.loop, self._dynip_resolve)
self.dynip_timer.start()
def __init__(self, loop):
self.loop = loop
self.watchers = []
ep = EVENT_ENDPOINT.split(':')
self.endpoint = (ep[0], int(ep[1]))
self.timers = []
self.stats_timer = pyev.Timer(3.0, 3.0, self.loop, self.print_stats)
self.stats_timer.start()
self.reqs = 0
self.resps = 0
self.no_resps = 0
self.errors = 0
self.conn_pool = [
EphemeralConnection(self.loop, self.endpoint, '', self.recv_http,
self.on_error, self.no_response)
for i in range(900)
]
self.conn_use = []
def send_event(self, buf, timeout):
# set the timeout to call the method that will
# create the connection and send the buffer (send buffer)
logging.debug('ad.send_event creating conn')
#conn = EphemeralConnection(
# self.loop, self.endpoint, buf, self.recv_http,
# self.on_error, self.no_response)
conn = None
try:
conn = self.conn_pool.pop()
self.conn_use.append(conn)
except:
logging.error('no more connections')
return
conn.buf = buf
conn.read_buf = ''
to = pyev.Timer(timeout, 0.0, self.loop, self.send_http, conn)
to.start()
self.timers.append(to)
logging.debug('ad.send_event creating conn done %d' % conn.id)
def connect(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setblocking(0)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
logging.info('connecting to %s:%d' %
(self.address[0], self.address[1]))
res = self.sock.connect_ex(self.address)
if res != errno.EINPROGRESS:
logging.error('unable to connect to %s:%d' %
(self.address[0], self.address[1]))
self.state = Connection.STATE_ERROR
else:
if not self.connect_cb:
self.connect_cb = self.io_cb
self.state = Connection.STATE_CONNECTING
self.watcher = pyev.Io(self.sock, pyev.EV_WRITE, self.loop,
self.connect_cb)
self.watcher.start()
# start the timer
self.timer = pyev.Timer(1, 1, self.loop, self.set_qps)
self.timer.start()
return self.state
def __init__(self, sock, address, ssl_config, loop, server):
"""Initialize the client socket.
:param sock: A socket object returned from server's accept() call.
:param address: Client's address as returned by server's accept() call.
:param ssl_config: A dictionary containing keyword arguments for
ssl.wrap_socket(). An empty dictionary will disable SSL.
:param loop: A pyev.Loop object that will handle signals and I/O.
:param server: The Server instance which created this client instance.
:raises: ssl.SSLError when failing to SSL wrap the socket
"""
self._sock = sock
self._address = address
self._server = server
self._sock.setblocking(0)
if ssl_config:
try:
self._sock = ssl.wrap_socket(self._sock, **ssl_config)
except ssl.SSLError:
logger.exception("refused {0}: SSL socket wrap error ".format(
self._address))
self._sock.close()
raise
else:
self._sock.setblocking(0)
self._read_watcher = pyev.Io(self._sock, pyev.EV_READ, loop,
self._read_handler)
self._write_watcher = pyev.Io(self._sock, pyev.EV_WRITE, loop,
self._write_handler)
self._timeout_watcher = pyev.Timer(server.timeout, server.timeout,
loop, self._timeout_handler)
self._read_watcher.start()
self._timeout_watcher.reset()
def init_duration(self):
if self.duration > 0:
self.watchers["duration"] = pyev.Timer(self.duration, 0.0,
self.loop, self.duration_cb)
def init_interval(self):
self.watchers["interval"] = pyev.Timer(self.interval, 0.0, self.loop,
self.interval_cb)
def init_interval(self):
if self.beacon_interval_sec > 0:
self.watchers["interval"] = pyev.Timer(self.beacon_interval_sec, 0.0, self.loop, self.interval_cb)
def call_later(self, delay, func, *args, **kwargs):
timer = pyev.Timer(delay, 0.0, loop, self._do_later,
(func, args, kwargs))
timer.start()
return timer
def stats_init(self):
self.add_calls = 0
self.last_stats = time.time()
self.stats_timer = pyev.Timer(2.0, 2.0, self.loop, self.stats)
self.stats_timer.start()
def sig_cb(watcher, events):
print("got SIGINT")
# optional - stop all watchers
if watcher.data:
print("stopping watchers: {0}".format(watcher.data))
while watcher.data:
watcher.data.pop().stop()
# unloop all nested loop
print("stopping the loop: {0}".format(watcher.loop))
watcher.loop.stop(pyev.EVBREAK_ALL)
def timer_cb(watcher, revents):
watcher.data += 1
print("timer.data: {0}".format(watcher.data))
print("timer.loop.iteration: {0}".format(watcher.loop.iteration))
print("timer.loop.now(): {0}".format(watcher.loop.now()))
if __name__ == "__main__":
loop = pyev.default_loop()
# initialise and start a repeating timer
timer = pyev.Timer(0, 2, loop, timer_cb, data=0)
timer.start()
# initialise and start a Signal watcher
sig = pyev.Signal(signal.SIGINT, loop, sig_cb)
sig.data = [timer, sig] # optional
sig.start()
# now wait for events to arrive
loop.start()
schedule_timer.start() def schedule_cb(w, e): l = len(scheduled_calls) c = 0 for cb, args, kwargs in scheduled_calls: try: cb(*args, **kwargs) except: traceback.print_exc() c += 1 if c == l: break del scheduled_calls[:c] schedule_timer = pyev.Timer(0.0, 0.0, default_loop, schedule_cb, data=None) def later(seconds, cb, *args, **kwargs): def wrap(watcher, events): args, kwargs = watcher.data timed_calls.remove(watcher) try: cb(*args, **kwargs) except: traceback.print_exc() t = pyev.Timer(seconds, 0.0, default_loop, wrap, data=(args, kwargs)) timed_calls.add(t) t.start()
def init_timeout(self):
if self.timeout > 0:
self.watchers["timeout"] = pyev.Timer(self.timeout, 0.0, self.loop,
self.timeout_cb)
import pyev
import time
def stop(watcher, events):
watcher.loop.stop()
def timed(watcher, events):
pass
loop = pyev.default_loop()
t1 = pyev.Timer(10.0, 0.0, loop, stop)
t2 = pyev.Timer(0.00000001, 0.00000001, loop, timed)
t1.start()
t2.start()
before = time.time()
loop.start()
after = time.time()
diff = after-before
ips = loop.iteration/diff
print("iterations per second: %f" % ips)
def __init__(self, loop, factory):
self.loop = loop
self.factory = factory
self.timer = pyev.Timer(2.0, 2.0, loop, self._print_stats)
def start(self):
logging.info("Event loop start")
self.watcher = pyev.Timer(self.config["interval_secs"], 1.0, self.loop, self.interval_cb)
self.watcher.start()
self.loop.start()
