def create_worker_thread(self): while connect_control.keep_running: try: ssl_sock = https_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # xlog.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_worke=False) except: time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 if idle_num > 5 and acceptable_num > 20: self.triger_create_worker_cv.wait()
def request(headers={}, payload=None): max_retry = 3 for i in range(max_retry): ssl_sock = None try: ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: xlog.debug('create_ssl_connection fail') continue if ssl_sock.host == '': ssl_sock.appid = appid_manager.get_appid() if not ssl_sock.appid: raise GAE_Exception(1, "no appid can use") headers['Host'] = ssl_sock.appid + ".appspot.com" ssl_sock.host = headers['Host'] else: headers['Host'] = ssl_sock.host response = _request(ssl_sock, headers, payload) if not response: ssl_sock.close() continue response.ssl_sock = ssl_sock return response except Exception as e: xlog.warn('request failed:%s', e) if ssl_sock: ssl_sock.close() raise GAE_Exception(2, "try max times")
def fetch(method, host, path, headers, payload, bufsize=8192): request_data = '%s %s HTTP/1.1\r\n' % (method, path) request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items()) request_data += '\r\n' ssl_sock = https_manager.get_ssl_connection(host) if not ssl_sock: return ssl_sock.send(request_data.encode()) payload_len = len(payload) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = ssl_sock.send(payload[start:start+send_size]) start += sended response = httplib.HTTPResponse(ssl_sock, buffering=True) response.ssl_sock = ssl_sock try: orig_timeout = ssl_sock.gettimeout() ssl_sock.settimeout(90) response.begin() ssl_sock.settimeout(orig_timeout) except httplib.BadStatusLine as e: xlog.warn("direct_handler.fetch bad status line:%r", e) response = None except Exception as e: xlog.warn("direct_handler.fetch:%r", e) return response
def get_worker(self): best_rtt = 9999 best_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 rtt = worker.get_rtt_rate() if rtt < best_rtt: best_rtt = rtt best_worker = worker if idle_num == 0 or len(self.workers) < self.min_worker_num: self.create_more_worker() if best_worker: return best_worker ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: raise GAE_Exception(1, "no ssl_sock") worker = self.on_ssl_created_cb(ssl_sock) return worker
def fetch(method, host, path, headers, payload, bufsize=8192): request_data = '%s %s HTTP/1.1\r\n' % (method, path) request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items()) request_data += '\r\n' ssl_sock = https_manager.get_ssl_connection(host) if not ssl_sock: return try: ssl_sock.send(request_data.encode()) payload_len = len(payload) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = ssl_sock.send(payload[start:start+send_size]) start += sended response = httplib.HTTPResponse(ssl_sock, buffering=True) response.ssl_sock = ssl_sock orig_timeout = ssl_sock.gettimeout() ssl_sock.settimeout(90) response.begin() ssl_sock.settimeout(orig_timeout) except httplib.BadStatusLine as e: xlog.warn("direct_handler.fetch bad status line:%r", e) google_ip.report_connect_closed(ssl_sock.ip, "request_fail") response = None except Exception as e: xlog.warn("direct_handler.fetch:%r", e) google_ip.report_connect_closed(ssl_sock.ip, "request_fail") response = None return response
def request(headers={}, payload=None): max_retry = 3 for i in range(max_retry): ssl_sock = None try: ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: xlog.debug("create_ssl_connection fail") continue if ssl_sock.host == "": ssl_sock.appid = appid_manager.get_appid() if not ssl_sock.appid: raise GAE_Exception(1, "no appid can use") headers["Host"] = ssl_sock.appid + ".appspot.com" ssl_sock.host = headers["Host"] else: headers["Host"] = ssl_sock.host response = _request(ssl_sock, headers, payload) if not response: ssl_sock.close() continue response.ssl_sock = ssl_sock return response except Exception as e: xlog.warn("request failed:%s", e) if ssl_sock: ssl_sock.close() raise GAE_Exception(2, "try max times")
def get_worker(self): best_rtt = 9999 best_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 rtt = worker.get_rtt_rate() if rtt < best_rtt: best_rtt = rtt best_worker = worker if idle_num == 0: self.create_more_worker() if best_worker: return best_worker ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: raise GAE_Exception(1, "no ssl_sock") worker = self.on_ssl_created_cb(ssl_sock) return worker
def request(headers={}, payload=None): max_retry = 3 for i in range(max_retry): ssl_sock = None try: ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: xlog.debug('create_ssl_connection fail') continue if ssl_sock.host == '': ssl_sock.appid = appid_manager.get_appid() if not ssl_sock.appid: google_ip.report_connect_closed(ssl_sock.ip, "no appid") raise GAE_Exception(1, "no appid can use") headers['Host'] = ssl_sock.appid + ".appspot.com" ssl_sock.host = headers['Host'] else: headers['Host'] = ssl_sock.host response = _request(ssl_sock, headers, payload) if not response: google_ip.report_connect_closed(ssl_sock.ip, "request_fail") ssl_sock.close() continue response.ssl_sock = ssl_sock return response except Exception as e: xlog.exception('request failed:%s', e) if ssl_sock: google_ip.report_connect_closed(ssl_sock.ip, "request_except") ssl_sock.close() raise GAE_Exception(2, "try max times")
def create_worker_thread(self): try: while True: ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: #xlog.warn("create_worker_thread get ssl_sock fail") continue self.on_ssl_created_cb(ssl_sock) if len(self.workers) > self.min_worker_num: break finally: self.create_worker_th = None
def create_worker_thread(self): ssl_sock = https_manager.get_ssl_connection() if not ssl_sock: xlog.warn("create_worker_thread get ssl_sock fail") self.on_ssl_created_cb(ssl_sock)