def get_ssl_connection(self, max_timeout=120): start_time = time.time() while True: if self.new_conn_pool.qsize() < self.connection_pool_min: self.create_more_connection() ret = self.new_conn_pool.get(True, 1) if ret: handshake_time, ssl_sock = ret if time.time( ) - ssl_sock.last_use_time > self.ssl_first_use_timeout: # xlog.debug("new_conn_pool.get:%s handshake:%d timeout.", ssl_sock.ip, handshake_time) ip_manager.report_connect_closed(ssl_sock.ip, "get_timeout") ssl_sock.close() continue else: xlog.debug("new_conn_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time) return ssl_sock else: if time.time() - start_time > max_timeout: xlog.debug("create ssl timeout fail.") return None
def get_ssl_connection(self, max_timeout=120): try: self.getter_num += 1 start_time = time.time() while True: if self.new_conn_pool.qsize() < self.connection_pool_min: self.create_more_connection() ret = self.new_conn_pool.get(True, 1) if ret: handshake_time, ssl_sock = ret if time.time() - ssl_sock.last_use_time > self.ssl_first_use_timeout: # xlog.debug("new_conn_pool.get:%s handshake:%d timeout.", ssl_sock.ip, handshake_time) ip_manager.report_connect_closed(ssl_sock.ip, "get_timeout") ssl_sock.close() continue else: # xlog.debug("new_conn_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time) return ssl_sock else: if time.time() - start_time > max_timeout: xlog.debug("create ssl timeout fail.") return None finally: self.getter_num -= 1
def head_request(self): # for keep alive, not work now. self.accept_task = False self.request_onway = True self.record_active("head") start_time = time.time() # xlog.debug("head request %s", self.ip) request_data = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % self.ssl_sock.host try: data = request_data.encode() ret = self.ssl_sock.send(data) if ret != len(data): xlog.warn("h1 head send len:%r %d %s", ret, len(data), self.ip) xlog.warn('%s trace:%s', self.ip, self.get_trace()) return False response = simple_http_client.Response(self.ssl_sock) response.begin(timeout=30) status = response.status if status != 200: xlog.warn("%s host:%s head fail status:%d", self.ip, self.ssl_sock.host, status) return False content = response.readall(timeout=5) time_now = time.time() self.record_active("head end") self.rtt = (time_now - start_time) * 1000 self.last_active_time = start_time ip_manager.update_ip(self.ip, self.rtt) self.accept_task = True return True except httplib.BadStatusLine as e: time_now = time.time() inactive_time = time_now - self.last_active_time head_timeout = time_now - start_time xlog.warn("%s keep alive fail, inactive_time:%d head_timeout:%d", self.ssl_sock.ip, inactive_time, head_timeout) xlog.warn('%s trace:%s', self.ip, self.get_trace()) ip_manager.report_connect_closed(self.ip, "down fail") self.close("head fail") except Exception as e: xlog.warn("h1 %s HEAD keep alive request fail:%r", self.ssl_sock.ip, e) xlog.warn('%s trace:%s', self.ip, self.get_trace()) ip_manager.report_connect_closed(self.ip, "down fail") self.close("head fail") finally: self.request_onway = False
def request_task(self, task): start_time = time.time() task.set_state("h1_req") task.headers['Host'] = self.task.host task.headers["Content-Length"] = len(task.body) request_data = '%s %s HTTP/1.1\r\n' % (task.method, task.path) request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in task.headers.items()) request_data += '\r\n' try: self.ssl_sock.send(request_data.encode()) payload_len = len(task.body) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = self.ssl_sock.send(task.body[start:start + send_size]) start += sended response = httplib.HTTPResponse(self.ssl_sock, buffering=True) self.ssl_sock.settimeout(100) response.begin() except Exception as e: xlog.warn("%s h1_request:%r inactive_time:%d", self.ip, e, time.time() - self.last_active_time) ip_manager.report_connect_closed(self.ssl_sock.ip, "request_fail") self.retry_task_cb(task) self.task = None self.close("request fail") return task.set_state("h1_get_head") if "transfer-encoding" in response.msg: length = 0 while True: try: data = response.read(8192) except httplib.IncompleteRead, e: data = e.partial except Exception as e: xlog.warn("Transfer-Encoding fail, ip:%s url:%s e:%r", self.ip, task.url, e) break if not data: break length += len(data) task.put_data(data)
def timeout_response(self): start_time = time.time() while time.time() - start_time < self.task.timeout: time.sleep(1) if self._remote_closed: return xlog.warn("h2 timeout %s task_trace:%s worker_trace:%s", self.connection.ssl_sock.ip, self.task.get_trace(), self.connection.get_trace()) self.task.set_state("timeout") if self.task.responsed: self.task.finish() else: self.task.response_fail("timeout") ip_manager.report_connect_closed(self.connection.ssl_sock.ip, "down fail") self.connection.close("timeout")
def keep_alive_thread(self): while connect_control.keep_running: if not connect_control.is_active(): time.sleep(1) continue to_keep_live_list = self.new_conn_pool.get_need_keep_alive(maxtime=self.ssl_first_use_timeout-2) for ssl_sock in to_keep_live_list: inactive_time = time.time() - ssl_sock.last_use_time if inactive_time > self.ssl_first_use_timeout + 5 or not self.ssl_timeout_cb: ip_manager.report_connect_closed(ssl_sock.ip, "alive_timeout") ssl_sock.close() else: # put ssl to worker try: self.ssl_timeout_cb(ssl_sock) except: # no appid avaiable pass time.sleep(1)
def request_task(self, task): timeout = task.timeout self.request_onway = True start_time = time.time() self.record_active("request") task.set_state("h1_req") task.headers['Host'] = self.task.host task.headers["Content-Length"] = len(task.body) request_data = '%s %s HTTP/1.1\r\n' % (task.method, task.path) request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in task.headers.items()) request_data += '\r\n' try: self.ssl_sock.send(request_data.encode()) payload_len = len(task.body) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = self.ssl_sock.send(task.body[start:start + send_size]) start += sended task.set_state("h1_req_sended") response = simple_http_client.Response(self.ssl_sock) response.begin(timeout=timeout) task.set_state("response_begin") except Exception as e: xlog.exception("%s h1_request:%r inactive_time:%d task.timeout:%d", self.ip, e, time.time() - self.last_active_time, task.timeout) xlog.warn('%s trace:%s', self.ip, self.get_trace()) ip_manager.report_connect_closed(self.ip, "down fail") self.retry_task_cb(task) self.task = None self.close("request fail") return task.set_state("h1_get_head") time_left = timeout - (time.time() - start_time) try: data = response.readall(timeout=time_left) except Exception as e: xlog.exception( "read fail, ip:%s, chunk:%d url:%s task.timeout:%d e:%r", self.ip, response.chunked, task.url, task.timeout, e) xlog.warn('%s trace:%s', self.ip, self.get_trace()) ip_manager.report_connect_closed(self.ip, "down fail") self.close("read fail") return response.worker = self response.task = task response.ssl_sock = self.ssl_sock length = len(data) task.content_length = length task.put_data(data) task.responsed = True task.queue.put(response) self.ssl_sock.received_size += length time_cost = (time.time() - start_time) if time_cost != 0: speed = length / time_cost task.set_state("h1_finish[SP:%d]" % speed) self.transfered_size += len(request_data) + length self.task = None self.accept_task = True self.idle_cb() self.processed_tasks += 1 self.last_active_time = time.time() self.record_active("Res")
self.ip, (time.time()-time_response)*1000, body_length, (end-start)) break else: time.sleep(0.1) continue last_read_time = time.time() data_len = len(data) start += data_len task.put_data(data) except Exception as e: xlog.warn("%s h1_request:%r", self.ip, e) task.put_data("") ip_manager.report_connect_closed(self.ssl_sock.ip, "down fail") self.close("request body fail") def head_request(self): # for keep alive start_time = time.time() xlog.debug("head request %s", self.ssl_sock.ip) request_data = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % self.ssl_sock.host try: data = request_data.encode() ret = self.ssl_sock.send(data) if ret != len(data): xlog.warn("h1 head send len:%r %d %s", ret, len(data), self.ip) return False
def request_task(self, task): timeout = task.timeout self.request_onway = True start_time = time.time() self.record_active("request") task.set_state("h1_req") task.headers['Host'] = self.task.host task.headers["Content-Length"] = len(task.body) request_data = '%s %s HTTP/1.1\r\n' % (task.method, task.path) request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in task.headers.items()) request_data += '\r\n' try: self.ssl_sock.send(request_data.encode()) payload_len = len(task.body) start = 0 while start < payload_len: send_size = min(payload_len - start, 65535) sended = self.ssl_sock.send(task.body[start:start+send_size]) start += sended task.set_state("h1_req_sended") response = simple_http_client.Response(self.ssl_sock) response.begin(timeout=timeout) task.set_state("response_begin") except Exception as e: xlog.exception("%s h1_request:%r inactive_time:%d task.timeout:%d", self.ip, e, time.time()-self.last_active_time, task.timeout) xlog.warn('%s trace:%s', self.ip, self.get_trace()) ip_manager.report_connect_closed(self.ip, "down fail") self.retry_task_cb(task) self.task = None self.close("request fail") return task.set_state("h1_get_head") time_left = timeout - (time.time() - start_time) try: data = response.readall(timeout=time_left) except Exception as e: xlog.exception("read fail, ip:%s, chunk:%d url:%s task.timeout:%d e:%r", self.ip, response.chunked, task.url, task.timeout, e) xlog.warn('%s trace:%s', self.ip, self.get_trace()) ip_manager.report_connect_closed(self.ip, "down fail") self.close("read fail") return response.worker = self response.task = task response.ssl_sock = self.ssl_sock length = len(data) task.content_length = length task.put_data(data) task.responsed = True task.queue.put(response) task.finish() self.ssl_sock.received_size += length time_cost = (time.time() - start_time) if time_cost != 0: speed = length / time_cost task.set_state("h1_finish[SP:%d]" % speed) self.transfered_size += len(request_data) + length self.task = None self.accept_task = True self.idle_cb() self.processed_tasks += 1 self.last_active_time = time.time() self.record_active("Res")