Ejemplo n.º 1
0
    def create_worker_thread(self):
        while connect_control.keep_running:
            try:
                ssl_sock = https_manager.get_ssl_connection()
            except Exception as e:
                continue

            if not ssl_sock:
                # xlog.warn("create_worker_thread get ssl_sock fail")
                continue

            try:
                self.on_ssl_created_cb(ssl_sock, check_free_worke=False)
            except:
                time.sleep(10)

            idle_num = 0
            acceptable_num = 0
            for worker in self.workers:
                if worker.accept_task:
                    acceptable_num += 1

                if worker.version == "1.1":
                    if worker.accept_task:
                        idle_num += 1
                else:
                    if len(worker.streams) == 0:
                        idle_num += 1

            if idle_num > 5 and acceptable_num > 20:
                self.triger_create_worker_cv.wait()
Ejemplo n.º 2
0
def request(headers={}, payload=None):
    max_retry = 3
    for i in range(max_retry):
        ssl_sock = None
        try:
            ssl_sock = https_manager.get_ssl_connection()
            if not ssl_sock:
                xlog.debug('create_ssl_connection fail')
                continue

            if ssl_sock.host == '':
                ssl_sock.appid = appid_manager.get_appid()
                if not ssl_sock.appid:
                    raise GAE_Exception(1, "no appid can use")
                headers['Host'] = ssl_sock.appid + ".appspot.com"
                ssl_sock.host = headers['Host']
            else:
                headers['Host'] = ssl_sock.host

            response = _request(ssl_sock, headers, payload)
            if not response:
                ssl_sock.close()
                continue

            response.ssl_sock = ssl_sock
            return response

        except Exception as e:
            xlog.warn('request failed:%s', e)
            if ssl_sock:
                ssl_sock.close()
    raise GAE_Exception(2, "try max times")
Ejemplo n.º 3
0
    def create_worker_thread(self):
        while connect_control.keep_running:
            try:
                ssl_sock = https_manager.get_ssl_connection()
            except Exception as e:
                continue

            if not ssl_sock:
                # xlog.warn("create_worker_thread get ssl_sock fail")
                continue

            try:
                self.on_ssl_created_cb(ssl_sock, check_free_worke=False)
            except:
                time.sleep(10)

            idle_num = 0
            acceptable_num = 0
            for worker in self.workers:
                if worker.accept_task:
                    acceptable_num += 1

                if worker.version == "1.1":
                    if worker.accept_task:
                        idle_num += 1
                else:
                    if len(worker.streams) == 0:
                        idle_num += 1

            if idle_num > 5 and acceptable_num > 20:
                self.triger_create_worker_cv.wait()
Ejemplo n.º 4
0
def fetch(method, host, path, headers, payload, bufsize=8192):
    request_data = '%s %s HTTP/1.1\r\n' % (method, path)
    request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items())
    request_data += '\r\n'

    ssl_sock = https_manager.get_ssl_connection(host)
    if not ssl_sock:
        return

    ssl_sock.send(request_data.encode())
    payload_len = len(payload)
    start = 0
    while start < payload_len:
        send_size = min(payload_len - start, 65535)
        sended = ssl_sock.send(payload[start:start+send_size])
        start += sended

    response = httplib.HTTPResponse(ssl_sock, buffering=True)

    response.ssl_sock = ssl_sock
    try:
        orig_timeout = ssl_sock.gettimeout()
        ssl_sock.settimeout(90)
        response.begin()
        ssl_sock.settimeout(orig_timeout)
    except httplib.BadStatusLine as e:
        xlog.warn("direct_handler.fetch bad status line:%r", e)
        response = None
    except Exception as e:
        xlog.warn("direct_handler.fetch:%r", e)
    return response
Ejemplo n.º 5
0
    def get_worker(self):
        best_rtt = 9999
        best_worker = None
        idle_num = 0
        for worker in self.workers:
            if not worker.accept_task:
                continue

            if worker.version == "1.1":
                idle_num += 1
            else:
                if len(worker.streams) == 0:
                    idle_num += 1

            rtt = worker.get_rtt_rate()

            if rtt < best_rtt:
                best_rtt = rtt
                best_worker = worker

        if idle_num == 0 or len(self.workers) < self.min_worker_num:
            self.create_more_worker()

        if best_worker:
            return best_worker

        ssl_sock = https_manager.get_ssl_connection()
        if not ssl_sock:
            raise GAE_Exception(1, "no ssl_sock")

        worker = self.on_ssl_created_cb(ssl_sock)

        return worker
Ejemplo n.º 6
0
def fetch(method, host, path, headers, payload, bufsize=8192):
    request_data = '%s %s HTTP/1.1\r\n' % (method, path)
    request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items())
    request_data += '\r\n'

    ssl_sock = https_manager.get_ssl_connection(host)
    if not ssl_sock:
        return

    try:
        ssl_sock.send(request_data.encode())
        payload_len = len(payload)
        start = 0
        while start < payload_len:
            send_size = min(payload_len - start, 65535)
            sended = ssl_sock.send(payload[start:start+send_size])
            start += sended

        response = httplib.HTTPResponse(ssl_sock, buffering=True)

        response.ssl_sock = ssl_sock

        orig_timeout = ssl_sock.gettimeout()
        ssl_sock.settimeout(90)
        response.begin()
        ssl_sock.settimeout(orig_timeout)
    except httplib.BadStatusLine as e:
        xlog.warn("direct_handler.fetch bad status line:%r", e)
        google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
        response = None
    except Exception as e:
        xlog.warn("direct_handler.fetch:%r", e)
        google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
        response = None
    return response
Ejemplo n.º 7
0
def request(headers={}, payload=None):
    max_retry = 3
    for i in range(max_retry):
        ssl_sock = None
        try:
            ssl_sock = https_manager.get_ssl_connection()
            if not ssl_sock:
                xlog.debug("create_ssl_connection fail")
                continue

            if ssl_sock.host == "":
                ssl_sock.appid = appid_manager.get_appid()
                if not ssl_sock.appid:
                    raise GAE_Exception(1, "no appid can use")
                headers["Host"] = ssl_sock.appid + ".appspot.com"
                ssl_sock.host = headers["Host"]
            else:
                headers["Host"] = ssl_sock.host

            response = _request(ssl_sock, headers, payload)
            if not response:
                ssl_sock.close()
                continue

            response.ssl_sock = ssl_sock
            return response

        except Exception as e:
            xlog.warn("request failed:%s", e)
            if ssl_sock:
                ssl_sock.close()
    raise GAE_Exception(2, "try max times")
Ejemplo n.º 8
0
    def get_worker(self):
        best_rtt = 9999
        best_worker = None
        idle_num = 0
        for worker in self.workers:
            if not worker.accept_task:
                continue

            if worker.version == "1.1":
                idle_num += 1
            else:
                if len(worker.streams) == 0:
                    idle_num += 1

            rtt = worker.get_rtt_rate()

            if rtt < best_rtt:
                best_rtt = rtt
                best_worker = worker

        if idle_num == 0:
            self.create_more_worker()

        if best_worker:
            return best_worker

        ssl_sock = https_manager.get_ssl_connection()
        if not ssl_sock:
            raise GAE_Exception(1, "no ssl_sock")

        worker = self.on_ssl_created_cb(ssl_sock)

        return worker
Ejemplo n.º 9
0
def request(headers={}, payload=None):
    max_retry = 3
    for i in range(max_retry):
        ssl_sock = None
        try:
            ssl_sock = https_manager.get_ssl_connection()
            if not ssl_sock:
                xlog.debug('create_ssl_connection fail')
                continue

            if ssl_sock.host == '':
                ssl_sock.appid = appid_manager.get_appid()
                if not ssl_sock.appid:
                    google_ip.report_connect_closed(ssl_sock.ip, "no appid")
                    raise GAE_Exception(1, "no appid can use")
                headers['Host'] = ssl_sock.appid + ".appspot.com"
                ssl_sock.host = headers['Host']
            else:
                headers['Host'] = ssl_sock.host

            response = _request(ssl_sock, headers, payload)
            if not response:
                google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
                ssl_sock.close()
                continue

            response.ssl_sock = ssl_sock
            return response

        except Exception as e:
            xlog.exception('request failed:%s', e)
            if ssl_sock:
                google_ip.report_connect_closed(ssl_sock.ip, "request_except")
                ssl_sock.close()
    raise GAE_Exception(2, "try max times")
Ejemplo n.º 10
0
    def create_worker_thread(self):
        try:
            while True:
                ssl_sock = https_manager.get_ssl_connection()
                if not ssl_sock:
                    #xlog.warn("create_worker_thread get ssl_sock fail")
                    continue

                self.on_ssl_created_cb(ssl_sock)

                if len(self.workers) > self.min_worker_num:
                    break

        finally:
            self.create_worker_th = None
Ejemplo n.º 11
0
    def create_worker_thread(self):
        try:
            while True:
                ssl_sock = https_manager.get_ssl_connection()
                if not ssl_sock:
                    #xlog.warn("create_worker_thread get ssl_sock fail")
                    continue

                self.on_ssl_created_cb(ssl_sock)

                if len(self.workers) > self.min_worker_num:
                    break

        finally:
            self.create_worker_th = None
Ejemplo n.º 12
0
    def create_worker_thread(self):
        ssl_sock = https_manager.get_ssl_connection()
        if not ssl_sock:
            xlog.warn("create_worker_thread get ssl_sock fail")

        self.on_ssl_created_cb(ssl_sock)
Ejemplo n.º 13
0
    def create_worker_thread(self):
        ssl_sock = https_manager.get_ssl_connection()
        if not ssl_sock:
            xlog.warn("create_worker_thread get ssl_sock fail")

        self.on_ssl_created_cb(ssl_sock)