def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout): ''' Constructor dsp_endpoints : is a list of tuples(endpoint, qps) where enpoint is a string like '192.168.10.152:5869' and qps is the value indicating queries per second for that enpoint. balance_conn_timeout : is the time period for rebalancing available connections. ''' # list containing tuples in the form # (endpoint, expected qps, current qps) self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints] self.event_endpoint = event_endpoint self.conns = {} self.awaiting_conns = {} self.event_conn_queue = [] self.event_conns = {} self.event_connections = 0 self.keep_alive_resp_waiting = {} self.balance_conn_to = balance_conn_timeout self.loop = pyev.default_loop() self.watchers = [ pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS ] self.watchers.append( pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop, self.balance)) self.watchers.append( pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop, self.check_established_connections)) self.watchers.append( pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop, self.check_pending_wins)) if EVENT_CONN_KEEP_ALIVE_TO: self.watchers.append( pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO, self.loop, self.send_keep_alives)) self.current_connections = 0 self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME) self.adserver = AdServer(self.loop) self.request_fact.initialize() self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver) if PLUGIN_DO_TO: self.watchers.append( pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop, self.request_fact.plugin_instance.do)) self.pending_wins = []
def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout): """ Constructor dsp_endpoints : is a list of tuples(endpoint, qps) where enpoint is a string like '192.168.10.152:5869' and qps is the value indicating queries per second for that enpoint. balance_conn_timeout : is the time period for rebalancing available connections. """ # list containing tuples in the form # (endpoint, expected qps, current qps) self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints] self.event_endpoint = event_endpoint self.conns = {} self.awaiting_conns = {} self.event_conn_queue = [] self.event_conns = {} self.event_connections = 0 self.keep_alive_resp_waiting = {} self.balance_conn_to = balance_conn_timeout self.loop = pyev.default_loop() self.watchers = [pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS] self.watchers.append(pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop, self.balance)) self.watchers.append(pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop, self.check_established_connections)) self.watchers.append(pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop, self.check_pending_wins)) if EVENT_CONN_KEEP_ALIVE_TO: self.watchers.append( pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO, self.loop, self.send_keep_alives) ) self.current_connections = 0 self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME) self.adserver = AdServer(self.loop) self.request_fact.initialize() self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver) if PLUGIN_DO_TO: self.watchers.append( pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop, self.request_fact.plugin_instance.do) ) self.pending_wins = []
class Exchange(object): def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout): """ Constructor dsp_endpoints : is a list of tuples(endpoint, qps) where enpoint is a string like '192.168.10.152:5869' and qps is the value indicating queries per second for that enpoint. balance_conn_timeout : is the time period for rebalancing available connections. """ # list containing tuples in the form # (endpoint, expected qps, current qps) self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints] self.event_endpoint = event_endpoint self.conns = {} self.awaiting_conns = {} self.event_conn_queue = [] self.event_conns = {} self.event_connections = 0 self.keep_alive_resp_waiting = {} self.balance_conn_to = balance_conn_timeout self.loop = pyev.default_loop() self.watchers = [pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS] self.watchers.append(pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop, self.balance)) self.watchers.append(pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop, self.check_established_connections)) self.watchers.append(pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop, self.check_pending_wins)) if EVENT_CONN_KEEP_ALIVE_TO: self.watchers.append( pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO, self.loop, self.send_keep_alives) ) self.current_connections = 0 self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME) self.adserver = AdServer(self.loop) self.request_fact.initialize() self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver) if PLUGIN_DO_TO: self.watchers.append( pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop, self.request_fact.plugin_instance.do) ) self.pending_wins = [] def signal_cb(self, watcher, revents): self.stop() self.adserver.stop() def stop(self): self.loop.stop(pyev.EVBREAK_ALL) while self.watchers: self.watchers.pop().stop() logging.debug("{0}: stopped".format(self)) self.adserver.stop() def start(self): """ Start watchers and loop """ # start generic watchers for times and signals for watcher in self.watchers: watcher.start() logging.debug("{0}: started".format(self)) self.loop.start() def balance(self, watcher, revents): """ Check your connections and balance """ logging.debug("balancing ...") for item in self.dest_eps: # load up endpoint = item[0] qps = item[1] # update current_qps if endpoint in self.conns: current = 0 for conn in self.conns[endpoint]: current += conn.last_qps item[2] = current current_qps = item[2] logging.info("qps=%d endpoint=%s conns=%d" % (current_qps, endpoint, self.current_connections)) # check if the endpoint is registered or # if the current qps is lower than expected if (endpoint not in self.conns) or (qps > current_qps): # we don't seem to have any connections or we have # not reached our expected qps yet, we should open # another connection if self.current_connections < MAX_CONNS: self.async_connect(endpoint) else: logging.warning("MAX_CONNS %d reached" % MAX_CONNS) # log wins per second if REPORT_WINS: wps = 0 for k, v in self.event_conns.iteritems(): wps += v.current_qps logging.info("wps=%d endpoint=%s conns=%d" % (wps, EVENT_ENDPOINT, len(self.event_conns))) def async_connect(self, endpoint): """ Asynchronously connect to an endpoint """ # create the connection logging.debug("launching async_connect to %s" % endpoint) ep = endpoint.split(":") ep = (ep[0], int(ep[1])) conn = Connection(ep, self.loop, self.create_request, self.receive_response, self.remove_connection, None) self.awaiting_conns[conn.id] = conn # create the entry if endpoint not in self.conns: self.conns[endpoint] = [] state = conn.connect() if state == Connection.STATE_CONNECTING: logging.debug("connecting!") def check_established_connections(self, watcher, revents): logging.debug("checking connections") # check if any of the awaiting_conns are done for cid, conn in self.awaiting_conns.items(): ep_key = ":".join([str(i) for i in conn.address]) if conn.state == Connection.STATE_CONNECTED: logging.info("connected to %s " % ep_key) logging.debug("bid connecting with id %d" % conn.id) # save the connection self.conns[ep_key].append(conn) # remove from awaiting_conns del self.awaiting_conns[cid] self.current_connections += 1 elif conn.state == Connection.STATE_CONNECTING: logging.info("still trying to connect to %s " % ep_key) else: logging.error("unable to connect to %s " % ep_key) # remove from awaiting_conns self.awaiting_conns[cid].close() del self.awaiting_conns[cid] logging.error("unable to connect to end") def remove_connection(self, conn): logging.error("removing connection %d" % conn.id) self.current_connections -= 1 ep_key = ":".join([str(i) for i in conn.address]) try: self.conns[ep_key].remove(conn) except ValueError: logging.info("connection %d was not yet persisted" % conn.id) def receive_response(self, read_buf, conn): logging.debug("ex.receive_response") buf, win, req_line, headers, body = self.request_fact.receive_response(read_buf) if (not buf) and win: # the buf was a full response and the # auction was won, call the request factory # to create a win request notification buf = self.request_fact.create_win_request(req_line, headers, body) # send it self.send_win_notification(buf) return "" elif buf and win is None: # this means that the buf was not a complete response return buf else: # this means that the buf was a complete response # but we did not win return "" def create_request(self, conn): logging.debug("ex.create_request") return self.request_fact.create_request() def send_win_notification(self, buf): logging.debug("ex.send_win_response") # do we have any connections available conn = self.get_event_connection() if not conn: logging.debug("ex.buffering win") self.pending_wins.append(buf) return # request an event to send the buffer conn.send_buffer(buf) def get_event_connection(self): logging.debug("ex.get_event_connection") # any connections left ? if len(self.event_conn_queue): return self.event_conn_queue.pop(0) # No available connections, can we create # another one? if not self.event_connections < MAX_EVENT_CONNS: return None # create another one ep = EVENT_ENDPOINT.split(":") ep = (ep[0], int(ep[1])) self.event_connections += 1 conn = Connection( ep, self.loop, self.create_win_request, self.receive_win_response, self.remove_event_connection, None ) state = conn.connect() if state == Connection.STATE_CONNECTING: logging.debug("event connecting with id %d" % conn.id) self.event_conns[conn.id] = conn return conn def create_win_request(self, conn): # we can pass since we set the buffer at # send_win_notification logging.error("This method should not ever be invoked") return "" def receive_win_response(self, read_buf, conn): logging.debug("ex.receive_win_response") self.event_conn_queue.append(conn) # set the connection into IDLE mode so that after # receiving the in response we don't register a # WRITE event conn.state = Connection.STATE_IDLE if conn.id in self.keep_alive_resp_waiting: del self.keep_alive_resp_waiting[conn.id] return "" return self.request_fact.receive_win_response(read_buf) def remove_event_connection(self, conn): logging.debug("ex.remove_event_connection %d", conn.id) try: self.event_connections -= 1 self.event_conn_queue.remove(conn) del self.event_conns[conn.id] except: logging.info("unable to remove event conn %d, it was not queued", conn.id) def check_pending_wins(self, watcher, revents): logging.debug("ex.check_pending_wins %d" % len(self.pending_wins)) # get the amount of idle connections wins = len(self.pending_wins) for i in range(wins): try: conn = self.get_event_connection() if conn: logging.debug("ex. sending pending win") conn.send_buffer(self.pending_wins.pop(0)) else: break except IndexError: break def send_keep_alives(self, watcher, revents): logging.debug("ex.send_keep_alives") buf = KEEP_ALIVE_HTTP_REQUEST % EVENT_CONN_KEEP_ALIVE_TO for i in range(len(self.event_conn_queue)): conn = self.event_conn_queue.pop(0) conn.send_buffer(buf) self.keep_alive_resp_waiting[conn.id] = conn
class Exchange(object): def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout): ''' Constructor dsp_endpoints : is a list of tuples(endpoint, qps) where enpoint is a string like '192.168.10.152:5869' and qps is the value indicating queries per second for that enpoint. balance_conn_timeout : is the time period for rebalancing available connections. ''' # list containing tuples in the form # (endpoint, expected qps, current qps) self.dest_eps = [ [ep[0], ep[1], 0] for ep in dsp_endpoints] self.event_endpoint = event_endpoint self.conns = {} self.awaiting_conns = {} self.event_conn_queue = [] self.event_conns = {} self.event_connections = 0 self.keep_alive_resp_waiting = {} self.balance_conn_to = balance_conn_timeout self.loop = pyev.default_loop() self.watchers = [pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS] self.watchers.append(pyev.Timer( self.balance_conn_to, self.balance_conn_to, self.loop, self.balance)) self.watchers.append(pyev.Timer( CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop, self.check_established_connections)) self.watchers.append(pyev.Timer( CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop, self.check_pending_wins)) if EVENT_CONN_KEEP_ALIVE_TO : self.watchers.append(pyev.Timer( EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO, self.loop, self.send_keep_alives)) self.current_connections = 0 self.request_fact = RTBRequestFactory( TEMPLATE_FILENAME) self.adserver = AdServer(self.loop) self.request_fact.initialize() self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver, PLUGIN_CONFIG) if PLUGIN_DO_TO: self.watchers.append(pyev.Timer( PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop, self.request_fact.plugin_instance.do)) self.pending_wins = [] def signal_cb(self, watcher, revents): self.stop() self.adserver.stop() def stop(self): self.loop.stop(pyev.EVBREAK_ALL) while self.watchers: self.watchers.pop().stop() logging.debug("{0}: stopped".format(self)) self.adserver.stop() def start(self): ''' Start watchers and loop ''' # start generic watchers for times and signals for watcher in self.watchers: watcher.start() logging.debug("{0}: started".format(self)) self.loop.start() def balance(self, watcher, revents): ''' Check your connections and balance ''' logging.debug('balancing ...') for item in self.dest_eps : #load up endpoint = item[0] qps = item[1] # update current_qps if endpoint in self.conns : current = 0 for conn in self.conns[endpoint]: current += conn.last_qps item[2] = current current_qps = item[2] logging.info('qps=%d endpoint=%s conns=%d' % (current_qps, endpoint, self.current_connections)) # check if the endpoint is registered or # if the current qps is lower than expected if (endpoint not in self.conns) or (qps > current_qps) : # we don't seem to have any connections or we have # not reached our expected qps yet, we should open # another connection if self.current_connections < MAX_CONNS : self.async_connect(endpoint) else : logging.warning('MAX_CONNS %d reached' % MAX_CONNS) # log wins per second if REPORT_WINS : wps = 0 for k,v in self.event_conns.iteritems(): wps += v.current_qps logging.info('wps=%d endpoint=%s conns=%d' % (wps, EVENT_ENDPOINT, len(self.event_conns))) def async_connect(self, endpoint): ''' Asynchronously connect to an endpoint ''' # create the connection logging.debug('launching async_connect to %s' % endpoint) ep = endpoint.split(':') ep = (ep[0], int(ep[1])) conn = Connection( ep, self.loop, self.create_request, self.receive_response, self.remove_connection, None) self.awaiting_conns[conn.id] = conn # create the entry if endpoint not in self.conns : self.conns[endpoint] = [] state = conn.connect() if state == Connection.STATE_CONNECTING: logging.debug('connecting!') def check_established_connections(self, watcher, revents): logging.debug('checking connections') # check if any of the awaiting_conns are done for cid, conn in self.awaiting_conns.items(): ep_key = ':'.join([str(i) for i in conn.address]) if conn.state == Connection.STATE_CONNECTED: logging.info('connected to %s ' % ep_key) logging.debug('bid connecting with id %d' % conn.id) # save the connection self.conns[ep_key].append(conn) # remove from awaiting_conns del self.awaiting_conns[cid] self.current_connections += 1 elif conn.state == Connection.STATE_CONNECTING: logging.info('still trying to connect to %s ' % ep_key) else : logging.error('unable to connect to %s ' % ep_key) # remove from awaiting_conns self.awaiting_conns[cid].close() del self.awaiting_conns[cid] logging.error('unable to connect to end') def remove_connection(self, conn): logging.error('removing connection %d' % conn.id) self.current_connections -= 1 ep_key = ':'.join([str(i) for i in conn.address]) try : self.conns[ep_key].remove(conn) except ValueError: logging.info('connection %d was not yet persisted' % conn.id) def receive_response(self, read_buf, conn): logging.debug('ex.receive_response') buf, win, req_line, headers, body = \ self.request_fact.receive_response(read_buf) if (not buf) and win : # the buf was a full response and the # auction was won, call the request factory # to create a win request notification buf = self.request_fact.create_win_request( req_line, headers, body) # send it self.send_win_notification(buf) return '' elif buf and win is None: # this means that the buf was not a complete response return buf else: # this means that the buf was a complete response # but we did not win return '' def create_request(self, conn): logging.debug('ex.create_request') return self.request_fact.create_request() def send_win_notification(self, buf): logging.debug('ex.send_win_response') # do we have any connections available conn = self.get_event_connection() if not conn: logging.debug('ex.buffering win') self.pending_wins.append(buf) return # request an event to send the buffer conn.send_buffer(buf) def get_event_connection(self): logging.debug('ex.get_event_connection') # any connections left ? if len(self.event_conn_queue): return self.event_conn_queue.pop(0) # No available connections, can we create # another one? if not self.event_connections < MAX_EVENT_CONNS: return None # create another one ep = EVENT_ENDPOINT.split(':') ep = (ep[0], int(ep[1])) self.event_connections += 1 conn = Connection( ep, self.loop, self.create_win_request, self.receive_win_response, self.remove_event_connection, None) state = conn.connect() if state == Connection.STATE_CONNECTING: logging.debug('event connecting with id %d' % conn.id) self.event_conns[conn.id] = conn return conn def create_win_request(self, conn): # we can pass since we set the buffer at # send_win_notification logging.error('This method should not ever be invoked') return '' def receive_win_response(self, read_buf, conn): logging.debug('ex.receive_win_response') self.event_conn_queue.append(conn) # set the connection into IDLE mode so that after # receiving the in response we don't register a # WRITE event conn.state = Connection.STATE_IDLE if conn.id in self.keep_alive_resp_waiting: del self.keep_alive_resp_waiting[conn.id] return '' return self.request_fact.receive_win_response(read_buf) def remove_event_connection(self, conn): logging.debug('ex.remove_event_connection %d', conn.id) try : self.event_connections -= 1 self.event_conn_queue.remove(conn) del self.event_conns[conn.id] except : logging.info( 'unable to remove event conn %d, it was not queued', conn.id) def check_pending_wins(self, watcher, revents): logging.debug('ex.check_pending_wins %d' % len(self.pending_wins)) # get the amount of idle connections wins = len(self.pending_wins) for i in range(wins): try : conn = self.get_event_connection() if conn : logging.debug('ex. sending pending win') conn.send_buffer(self.pending_wins.pop(0)) else : break except IndexError: break def send_keep_alives(self, watcher, revents): logging.debug('ex.send_keep_alives') buf = KEEP_ALIVE_HTTP_REQUEST % EVENT_CONN_KEEP_ALIVE_TO for i in range(len(self.event_conn_queue)): conn = self.event_conn_queue.pop(0) conn.send_buffer(buf) self.keep_alive_resp_waiting[conn.id] = conn