Exemplo n.º 1
0
    def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout):
        '''
            Constructor
            dsp_endpoints : is a list of tuples(endpoint, qps) where
                enpoint is a string like '192.168.10.152:5869'
                and qps is the value indicating queries per
                second for that enpoint.
            balance_conn_timeout : is the time period for rebalancing
                available connections.
        '''
        # list containing tuples in the form
        # (endpoint, expected qps, current qps)
        self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints]
        self.event_endpoint = event_endpoint
        self.conns = {}
        self.awaiting_conns = {}
        self.event_conn_queue = []
        self.event_conns = {}
        self.event_connections = 0
        self.keep_alive_resp_waiting = {}
        self.balance_conn_to = balance_conn_timeout
        self.loop = pyev.default_loop()
        self.watchers = [
            pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS
        ]

        self.watchers.append(
            pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop,
                       self.balance))

        self.watchers.append(
            pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop,
                       self.check_established_connections))

        self.watchers.append(
            pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop,
                       self.check_pending_wins))

        if EVENT_CONN_KEEP_ALIVE_TO:
            self.watchers.append(
                pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO,
                           self.loop, self.send_keep_alives))

        self.current_connections = 0
        self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME)
        self.adserver = AdServer(self.loop)
        self.request_fact.initialize()
        self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver)
        if PLUGIN_DO_TO:
            self.watchers.append(
                pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop,
                           self.request_fact.plugin_instance.do))
        self.pending_wins = []
Exemplo n.º 2
0
    def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout):
        """
            Constructor
            dsp_endpoints : is a list of tuples(endpoint, qps) where
                enpoint is a string like '192.168.10.152:5869'
                and qps is the value indicating queries per
                second for that enpoint.
            balance_conn_timeout : is the time period for rebalancing
                available connections.
        """
        # list containing tuples in the form
        # (endpoint, expected qps, current qps)
        self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints]
        self.event_endpoint = event_endpoint
        self.conns = {}
        self.awaiting_conns = {}
        self.event_conn_queue = []
        self.event_conns = {}
        self.event_connections = 0
        self.keep_alive_resp_waiting = {}
        self.balance_conn_to = balance_conn_timeout
        self.loop = pyev.default_loop()
        self.watchers = [pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS]

        self.watchers.append(pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop, self.balance))

        self.watchers.append(pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop, self.check_established_connections))

        self.watchers.append(pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop, self.check_pending_wins))

        if EVENT_CONN_KEEP_ALIVE_TO:
            self.watchers.append(
                pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO, self.loop, self.send_keep_alives)
            )

        self.current_connections = 0
        self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME)
        self.adserver = AdServer(self.loop)
        self.request_fact.initialize()
        self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver)
        if PLUGIN_DO_TO:
            self.watchers.append(
                pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop, self.request_fact.plugin_instance.do)
            )
        self.pending_wins = []
Exemplo n.º 3
0
class Exchange(object):
    def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout):
        """
            Constructor
            dsp_endpoints : is a list of tuples(endpoint, qps) where
                enpoint is a string like '192.168.10.152:5869'
                and qps is the value indicating queries per
                second for that enpoint.
            balance_conn_timeout : is the time period for rebalancing
                available connections.
        """
        # list containing tuples in the form
        # (endpoint, expected qps, current qps)
        self.dest_eps = [[ep[0], ep[1], 0] for ep in dsp_endpoints]
        self.event_endpoint = event_endpoint
        self.conns = {}
        self.awaiting_conns = {}
        self.event_conn_queue = []
        self.event_conns = {}
        self.event_connections = 0
        self.keep_alive_resp_waiting = {}
        self.balance_conn_to = balance_conn_timeout
        self.loop = pyev.default_loop()
        self.watchers = [pyev.Signal(sig, self.loop, self.signal_cb) for sig in STOPSIGNALS]

        self.watchers.append(pyev.Timer(self.balance_conn_to, self.balance_conn_to, self.loop, self.balance))

        self.watchers.append(pyev.Timer(CHECK_CONNS_TO, CHECK_CONNS_TO, self.loop, self.check_established_connections))

        self.watchers.append(pyev.Timer(CHECK_PENDING_TO, CHECK_PENDING_TO, self.loop, self.check_pending_wins))

        if EVENT_CONN_KEEP_ALIVE_TO:
            self.watchers.append(
                pyev.Timer(EVENT_CONN_KEEP_ALIVE_TO, EVENT_CONN_KEEP_ALIVE_TO, self.loop, self.send_keep_alives)
            )

        self.current_connections = 0
        self.request_fact = RTBRequestFactory(TEMPLATE_FILENAME)
        self.adserver = AdServer(self.loop)
        self.request_fact.initialize()
        self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver)
        if PLUGIN_DO_TO:
            self.watchers.append(
                pyev.Timer(PLUGIN_DO_TO, PLUGIN_DO_TO, self.loop, self.request_fact.plugin_instance.do)
            )
        self.pending_wins = []

    def signal_cb(self, watcher, revents):
        self.stop()
        self.adserver.stop()

    def stop(self):
        self.loop.stop(pyev.EVBREAK_ALL)
        while self.watchers:
            self.watchers.pop().stop()
        logging.debug("{0}: stopped".format(self))
        self.adserver.stop()

    def start(self):
        """
            Start watchers and loop
        """
        # start generic watchers for times and signals
        for watcher in self.watchers:
            watcher.start()
        logging.debug("{0}: started".format(self))
        self.loop.start()

    def balance(self, watcher, revents):
        """
            Check your connections and balance
        """
        logging.debug("balancing ...")

        for item in self.dest_eps:
            # load up
            endpoint = item[0]
            qps = item[1]
            # update current_qps
            if endpoint in self.conns:
                current = 0
                for conn in self.conns[endpoint]:
                    current += conn.last_qps
                item[2] = current
            current_qps = item[2]
            logging.info("qps=%d endpoint=%s conns=%d" % (current_qps, endpoint, self.current_connections))
            # check if the endpoint is registered or
            # if the current qps is lower than expected
            if (endpoint not in self.conns) or (qps > current_qps):
                # we don't seem to have any connections or we have
                # not reached our expected qps yet, we should open
                # another connection
                if self.current_connections < MAX_CONNS:
                    self.async_connect(endpoint)
                else:
                    logging.warning("MAX_CONNS %d reached" % MAX_CONNS)
        # log wins per second
        if REPORT_WINS:
            wps = 0
            for k, v in self.event_conns.iteritems():
                wps += v.current_qps
            logging.info("wps=%d endpoint=%s conns=%d" % (wps, EVENT_ENDPOINT, len(self.event_conns)))

    def async_connect(self, endpoint):
        """
            Asynchronously connect to an endpoint
        """
        # create the connection
        logging.debug("launching async_connect to %s" % endpoint)
        ep = endpoint.split(":")
        ep = (ep[0], int(ep[1]))
        conn = Connection(ep, self.loop, self.create_request, self.receive_response, self.remove_connection, None)
        self.awaiting_conns[conn.id] = conn
        # create the entry
        if endpoint not in self.conns:
            self.conns[endpoint] = []
        state = conn.connect()
        if state == Connection.STATE_CONNECTING:
            logging.debug("connecting!")

    def check_established_connections(self, watcher, revents):
        logging.debug("checking connections")
        # check if any of the awaiting_conns are done
        for cid, conn in self.awaiting_conns.items():
            ep_key = ":".join([str(i) for i in conn.address])
            if conn.state == Connection.STATE_CONNECTED:
                logging.info("connected to %s " % ep_key)
                logging.debug("bid connecting with id %d" % conn.id)
                # save the connection
                self.conns[ep_key].append(conn)
                # remove from awaiting_conns
                del self.awaiting_conns[cid]
                self.current_connections += 1
            elif conn.state == Connection.STATE_CONNECTING:
                logging.info("still trying to connect to %s " % ep_key)
            else:
                logging.error("unable to connect to %s " % ep_key)
                # remove from awaiting_conns
                self.awaiting_conns[cid].close()
                del self.awaiting_conns[cid]
                logging.error("unable to connect to end")

    def remove_connection(self, conn):
        logging.error("removing connection %d" % conn.id)
        self.current_connections -= 1
        ep_key = ":".join([str(i) for i in conn.address])
        try:
            self.conns[ep_key].remove(conn)
        except ValueError:
            logging.info("connection %d was not yet persisted" % conn.id)

    def receive_response(self, read_buf, conn):
        logging.debug("ex.receive_response")
        buf, win, req_line, headers, body = self.request_fact.receive_response(read_buf)
        if (not buf) and win:
            # the buf was a full response and the
            # auction was won, call the request factory
            # to create a win request notification
            buf = self.request_fact.create_win_request(req_line, headers, body)
            # send it
            self.send_win_notification(buf)
            return ""
        elif buf and win is None:
            # this means that the buf was not a complete response
            return buf
        else:
            # this means that the buf was  a complete response
            # but we did not win
            return ""

    def create_request(self, conn):
        logging.debug("ex.create_request")
        return self.request_fact.create_request()

    def send_win_notification(self, buf):
        logging.debug("ex.send_win_response")
        # do we have any connections available
        conn = self.get_event_connection()
        if not conn:
            logging.debug("ex.buffering win")
            self.pending_wins.append(buf)
            return
        # request an event to send the buffer
        conn.send_buffer(buf)

    def get_event_connection(self):
        logging.debug("ex.get_event_connection")
        # any connections left ?
        if len(self.event_conn_queue):
            return self.event_conn_queue.pop(0)
        # No available connections, can we create
        # another one?
        if not self.event_connections < MAX_EVENT_CONNS:
            return None
        # create another one
        ep = EVENT_ENDPOINT.split(":")
        ep = (ep[0], int(ep[1]))
        self.event_connections += 1
        conn = Connection(
            ep, self.loop, self.create_win_request, self.receive_win_response, self.remove_event_connection, None
        )
        state = conn.connect()
        if state == Connection.STATE_CONNECTING:
            logging.debug("event connecting with id %d" % conn.id)
        self.event_conns[conn.id] = conn
        return conn

    def create_win_request(self, conn):
        # we can pass since we set the buffer at
        # send_win_notification
        logging.error("This method should not ever be invoked")
        return ""

    def receive_win_response(self, read_buf, conn):
        logging.debug("ex.receive_win_response")
        self.event_conn_queue.append(conn)
        # set the connection into IDLE mode so that after
        # receiving the in response we don't register a
        # WRITE event
        conn.state = Connection.STATE_IDLE
        if conn.id in self.keep_alive_resp_waiting:
            del self.keep_alive_resp_waiting[conn.id]
            return ""
        return self.request_fact.receive_win_response(read_buf)

    def remove_event_connection(self, conn):
        logging.debug("ex.remove_event_connection %d", conn.id)
        try:
            self.event_connections -= 1
            self.event_conn_queue.remove(conn)
            del self.event_conns[conn.id]
        except:
            logging.info("unable to remove event conn %d, it was not queued", conn.id)

    def check_pending_wins(self, watcher, revents):
        logging.debug("ex.check_pending_wins %d" % len(self.pending_wins))
        # get the amount of idle connections
        wins = len(self.pending_wins)
        for i in range(wins):
            try:
                conn = self.get_event_connection()
                if conn:
                    logging.debug("ex. sending pending win")
                    conn.send_buffer(self.pending_wins.pop(0))
                else:
                    break
            except IndexError:
                break

    def send_keep_alives(self, watcher, revents):
        logging.debug("ex.send_keep_alives")
        buf = KEEP_ALIVE_HTTP_REQUEST % EVENT_CONN_KEEP_ALIVE_TO
        for i in range(len(self.event_conn_queue)):
            conn = self.event_conn_queue.pop(0)
            conn.send_buffer(buf)
            self.keep_alive_resp_waiting[conn.id] = conn
Exemplo n.º 4
0
class Exchange(object):

    def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout):
        '''
            Constructor
            dsp_endpoints : is a list of tuples(endpoint, qps) where
                enpoint is a string like '192.168.10.152:5869'
                and qps is the value indicating queries per
                second for that enpoint.
            balance_conn_timeout : is the time period for rebalancing
                available connections.
        '''
        # list containing tuples in the form 
        # (endpoint, expected qps, current qps)
        self.dest_eps = [ [ep[0], ep[1], 0] for ep in dsp_endpoints]
        self.event_endpoint = event_endpoint
        self.conns = {}
        self.awaiting_conns = {}
        self.event_conn_queue = []
        self.event_conns = {}
        self.event_connections = 0
        self.keep_alive_resp_waiting = {}        
        self.balance_conn_to = balance_conn_timeout
        self.loop = pyev.default_loop()
        self.watchers = [pyev.Signal(sig, self.loop, self.signal_cb)
                         for sig in STOPSIGNALS]

        self.watchers.append(pyev.Timer(
                                self.balance_conn_to, 
                                self.balance_conn_to, 
                                self.loop,
                                self.balance))

        self.watchers.append(pyev.Timer(
                                CHECK_CONNS_TO, 
                                CHECK_CONNS_TO, 
                                self.loop,
                                self.check_established_connections))

        self.watchers.append(pyev.Timer(
                                CHECK_PENDING_TO, 
                                CHECK_PENDING_TO, 
                                self.loop,
                                self.check_pending_wins))
        
        if EVENT_CONN_KEEP_ALIVE_TO :
            self.watchers.append(pyev.Timer(
                                    EVENT_CONN_KEEP_ALIVE_TO, 
                                    EVENT_CONN_KEEP_ALIVE_TO, 
                                    self.loop,
                                    self.send_keep_alives))

        self.current_connections = 0
        self.request_fact = RTBRequestFactory(
                                    TEMPLATE_FILENAME)
        self.adserver = AdServer(self.loop)
        self.request_fact.initialize()
        self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, 
                                             self.adserver,
                                             PLUGIN_CONFIG)
        if PLUGIN_DO_TO:
            self.watchers.append(pyev.Timer(
                                    PLUGIN_DO_TO, 
                                    PLUGIN_DO_TO, 
                                    self.loop,
                                    self.request_fact.plugin_instance.do))
        self.pending_wins = []
        

    def signal_cb(self, watcher, revents):
        self.stop()
        self.adserver.stop()

    def stop(self):
        self.loop.stop(pyev.EVBREAK_ALL)
        while self.watchers:
            self.watchers.pop().stop()
        logging.debug("{0}: stopped".format(self))
        self.adserver.stop()

    def start(self):
        '''
            Start watchers and loop
        '''
        # start generic watchers for times and signals        
        for watcher in self.watchers:
            watcher.start()
        logging.debug("{0}: started".format(self))
        self.loop.start()
        
    def balance(self, watcher, revents):
        '''
            Check your connections and balance
        '''
        logging.debug('balancing ...')
        
        for item in self.dest_eps :
            #load up
            endpoint = item[0]            
            qps = item[1]
            # update current_qps
            if endpoint in self.conns :
                current = 0
                for conn in self.conns[endpoint]:
                    current += conn.last_qps 
                item[2] = current
            current_qps = item[2]
            logging.info('qps=%d endpoint=%s conns=%d' % 
                            (current_qps, endpoint, self.current_connections))
            # check if the endpoint is registered or
            # if the current qps is lower than expected
            if (endpoint not in self.conns) or (qps > current_qps) :
                # we don't seem to have any connections or we have
                # not reached our expected qps yet, we should open
                # another connection
                if self.current_connections < MAX_CONNS :
                    self.async_connect(endpoint)
                else :
                    logging.warning('MAX_CONNS %d reached' % MAX_CONNS)
        # log wins per second
        if REPORT_WINS :
            wps = 0
            for k,v in self.event_conns.iteritems():
                wps += v.current_qps
            logging.info('wps=%d endpoint=%s conns=%d' % 
                                (wps, EVENT_ENDPOINT, len(self.event_conns)))

    def async_connect(self, endpoint):
        '''
            Asynchronously connect to an endpoint
        '''
        # create the connection
        logging.debug('launching async_connect to %s' % endpoint)
        ep = endpoint.split(':')
        ep = (ep[0], int(ep[1]))
        conn = Connection(
                ep, 
                self.loop, 
                self.create_request, 
                self.receive_response, 
                self.remove_connection,
                None)
        self.awaiting_conns[conn.id] = conn
        # create the entry
        if endpoint not in self.conns :
            self.conns[endpoint] = []
        state = conn.connect()
        if state == Connection.STATE_CONNECTING:
           logging.debug('connecting!')

    def check_established_connections(self, watcher, revents):
        logging.debug('checking connections')
        # check if any of the awaiting_conns are done
        for cid, conn in self.awaiting_conns.items():
            ep_key = ':'.join([str(i) for i in conn.address])
            if conn.state == Connection.STATE_CONNECTED:
                logging.info('connected to %s ' % ep_key)
                logging.debug('bid connecting with id %d' % conn.id)        
                # save the connection                    
                self.conns[ep_key].append(conn)
                # remove from awaiting_conns
                del self.awaiting_conns[cid]
                self.current_connections += 1
            elif conn.state == Connection.STATE_CONNECTING:
                logging.info('still trying to connect to %s ' % ep_key)
            else :
                logging.error('unable to connect to %s ' % ep_key)
                # remove from awaiting_conns
                self.awaiting_conns[cid].close()
                del self.awaiting_conns[cid]
                logging.error('unable to connect to end')


    def remove_connection(self, conn):
        logging.error('removing connection %d' % conn.id)
        self.current_connections -= 1
        ep_key = ':'.join([str(i) for i in conn.address])
        try :
            self.conns[ep_key].remove(conn)
        except ValueError:
            logging.info('connection %d was not yet persisted' % conn.id)

    def receive_response(self, read_buf, conn):
        logging.debug('ex.receive_response')
        buf, win, req_line, headers, body = \
            self.request_fact.receive_response(read_buf)
        if (not buf) and win :
            # the buf was a full response and the 
            # auction was won, call the request factory
            # to create a win request notification
            buf = self.request_fact.create_win_request(
                                            req_line, headers, body)
            # send it
            self.send_win_notification(buf)
            return ''
        elif buf and win is None:
            # this means that the buf was not a complete response
            return buf
        else:
            # this means that the buf was  a complete response
            # but we did not win
            return ''

    def create_request(self, conn):
        logging.debug('ex.create_request')
        return self.request_fact.create_request()

    def send_win_notification(self, buf):
        logging.debug('ex.send_win_response')
        # do we have any connections available
        conn = self.get_event_connection()
        if not conn:
            logging.debug('ex.buffering win')
            self.pending_wins.append(buf)
            return
        # request an event to send the buffer
        conn.send_buffer(buf)

    def get_event_connection(self):
        logging.debug('ex.get_event_connection')
        # any connections left ?
        if len(self.event_conn_queue):
            return self.event_conn_queue.pop(0)
        # No available connections, can we create 
        # another one?
        if not self.event_connections < MAX_EVENT_CONNS:
            return None
        # create another one
        ep = EVENT_ENDPOINT.split(':')
        ep = (ep[0], int(ep[1]))
        self.event_connections += 1
        conn = Connection(
                ep, 
                self.loop, 
                self.create_win_request, 
                self.receive_win_response, 
                self.remove_event_connection,
                None)
        state = conn.connect()
        if state == Connection.STATE_CONNECTING:
           logging.debug('event connecting with id %d' % conn.id)
        self.event_conns[conn.id] = conn       
        return conn
    
    def create_win_request(self, conn):
        # we can pass since we set the buffer at
        # send_win_notification
        logging.error('This method should not ever be invoked')
        return ''

    def receive_win_response(self, read_buf, conn):
        logging.debug('ex.receive_win_response')
        self.event_conn_queue.append(conn)
        # set the connection into IDLE mode so that after
        # receiving the in response we don't register a
        # WRITE event
        conn.state = Connection.STATE_IDLE
        if conn.id in self.keep_alive_resp_waiting:
            del self.keep_alive_resp_waiting[conn.id]
            return ''
        return self.request_fact.receive_win_response(read_buf) 

    def remove_event_connection(self, conn):
        logging.debug('ex.remove_event_connection %d', conn.id)
        try :
            self.event_connections -= 1
            self.event_conn_queue.remove(conn)
            del self.event_conns[conn.id]
        except :
            logging.info(
                'unable to remove event conn %d, it was not queued',
                 conn.id)

    def check_pending_wins(self,  watcher, revents):
        logging.debug('ex.check_pending_wins %d' % len(self.pending_wins))
        # get the amount of idle connections        
        wins = len(self.pending_wins)
        for i in range(wins):
            try :
                conn = self.get_event_connection()
                if conn :
                    logging.debug('ex. sending pending win')
                    conn.send_buffer(self.pending_wins.pop(0))
                else :
                    break
            except IndexError:
                break

    def send_keep_alives(self,  watcher, revents):
        logging.debug('ex.send_keep_alives')
        buf = KEEP_ALIVE_HTTP_REQUEST % EVENT_CONN_KEEP_ALIVE_TO
        for i in range(len(self.event_conn_queue)):
            conn = self.event_conn_queue.pop(0)
            conn.send_buffer(buf)
            self.keep_alive_resp_waiting[conn.id] = conn