def __init__(self, logger, config, ip_manager, connection_manager, http1worker=Http1Worker, http2worker=Http2Worker): self.logger = logger self.config = config self.ip_manager = ip_manager self.connection_manager = connection_manager self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb) self.http1worker = http1worker self.http2worker = http2worker self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.last_request_time = time.time() self.running = True self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = simple_queue.Queue() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start()
def __init__(self, logger, config, ip_manager, connection_manager, http1worker=Http1Worker, http2worker=Http2Worker): self.logger = logger self.config = config self.ip_manager = ip_manager self.connection_manager = connection_manager self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb) self.http1worker = http1worker self.http2worker = http2worker self.request_queue = queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.last_request_time = time.time() self.task_count_lock = threading.Lock() self.task_count = 0 self.running = True # for statistic self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = [] self.last_sent = self.total_sent = 0 self.last_received = self.total_received = 0 self.second_stats = queue.deque() self.last_statistic_time = time.time() self.second_stat = { "rtt": 0, "sent": 0, "received": 0 } self.minute_stat = { "rtt": 0, "sent": 0, "received": 0 } self.trigger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = simple_queue.Queue() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() threading.Thread(target=self.connection_checker).start()
def __init__(self): self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.create_worker_th = None self.last_request_time = time.time() self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = SimpleCondition() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() # move created ssl to worker after ssl timeout https_manager.set_ssl_time_handler(self.on_ssl_created_cb)
def __init__(self, host, log_debug_data): self.host = host self.log_debug_data = log_debug_data self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.create_worker_th = None self.last_request_time = time.time() self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = SimpleCondition() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() # move created ssl to worker after ssl timeout self.https_manager = connect_manager.Https_connection_manager(host, self.on_ssl_created_cb)
def __init__(self, logger, config, ip_manager, connection_manager, http1worker=Http1Worker, http2worker=Http2Worker): self.logger = logger self.config = config self.ip_manager = ip_manager self.connection_manager = connection_manager self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb) self.http1worker = http1worker self.http2worker = http2worker self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.last_request_time = time.time() self.task_count_lock = threading.Lock() self.task_count = 0 self.running = True # for statistic self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = [] self.last_sent = self.total_sent = 0 self.last_received = self.total_received = 0 self.second_stats = Queue.deque() self.last_statistic_time = time.time() self.second_stat = { "rtt": 0, "sent": 0, "received": 0 } self.minute_stat = { "rtt": 0, "sent": 0, "received": 0 } self.trigger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = simple_queue.Queue() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() threading.Thread(target=self.connection_checker).start()
class HttpsDispatcher(object): idle_time = 2 * 60 def __init__(self, logger, config, ip_manager, connection_manager, http1worker=Http1Worker, http2worker=Http2Worker): self.logger = logger self.config = config self.ip_manager = ip_manager self.connection_manager = connection_manager self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb) self.http1worker = http1worker self.http2worker = http2worker self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.last_request_time = time.time() self.task_count_lock = threading.Lock() self.task_count = 0 self.running = True # for statistic self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = [] self.last_sent = self.total_sent = 0 self.last_received = self.total_received = 0 self.second_stats = Queue.deque() self.last_statistic_time = time.time() self.second_stat = { "rtt": 0, "sent": 0, "received": 0 } self.minute_stat = { "rtt": 0, "sent": 0, "received": 0 } self.trigger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = simple_queue.Queue() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() threading.Thread(target=self.connection_checker).start() def stop(self): self.running = False self.request_queue.put(None) self.close_all_worker("stop") def on_ssl_created_cb(self, ssl_sock, check_free_work=True): # self.logger.debug("on_ssl_created_cb %s", ssl_sock.ip) if not self.running: ssl_sock.close() return if not ssl_sock: raise Exception("on_ssl_created_cb ssl_sock None") if ssl_sock.h2: worker = self.http2worker( self.logger, self.ip_manager, self.config, ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h2_num += 1 else: worker = self.http1worker( self.logger, self.ip_manager, self.config, ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h1_num += 1 self.workers.append(worker) if check_free_work: self.check_free_worker() def _on_worker_idle_cb(self): self.wait_a_worker_cv.notify() def create_worker_thread(self): while self.running: self.trigger_create_worker_cv.wait() try: ssl_sock = self.connection_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # self.logger.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_work=False) except: time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 def get_worker(self, nowait=False): while self.running: best_score = 99999999 best_worker = None idle_num = 0 now = time.time() for worker in self.workers: if not worker.accept_task: # self.logger.debug("not accept") continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 score = worker.get_score() if best_score > score: best_score = score best_worker = worker if len(self.workers) < self.config.dispather_max_workers and \ (best_worker is None or idle_num < self.config.dispather_min_idle_workers or len(self.workers) < self.config.dispather_min_workers or (now - best_worker.last_recv_time) < self.config.dispather_work_min_idle_time or best_score > self.config.dispather_work_max_score or (best_worker.version == "2" and len(best_worker.streams) >= self.config.http2_target_concurrent)): # self.logger.debug("trigger get more worker") self.trigger_create_worker_cv.notify() if nowait or \ (best_worker and (now - best_worker.last_recv_time) >= self.config.dispather_work_min_idle_time): # self.logger.debug("return worker") return best_worker self.wait_a_worker_cv.wait(time.time() + 1) # self.logger.debug("get wait_a_worker_cv") #time.sleep(0.1) def check_free_worker(self): # close slowest worker, # give change for better worker while True: slowest_score = 9999 slowest_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "2" and len(worker.streams) > 0: continue score = worker.get_score() if score < 1000: idle_num += 1 if score > slowest_score: slowest_score = score slowest_worker = worker if idle_num < self.config.dispather_max_idle_workers or \ idle_num < int(len(self.workers) * 0.3) or \ len(self.workers) < self.config.dispather_max_workers: return if slowest_worker is None: return self.close_cb(slowest_worker) def request(self, method, host, path, headers, body, url="", timeout=60): if self.task_count > self.config.max_task_num: self.logger.warn("task num exceed") time.sleep(1) return None with self.task_count_lock: self.task_count += 1 try: # self.logger.debug("task start request") if not url: url = "%s %s%s" % (method, host, path) self.last_request_time = time.time() q = simple_queue.Queue() task = http_common.Task(self.logger, self.config, method, host, path, headers, body, q, url, timeout) task.set_state("start_request") self.request_queue.put(task) response = q.get(timeout=timeout) if response and response.status == 200: self.success_num += 1 self.continue_fail_num = 0 else: self.logger.warn("task %s %s %s timeout", method, host, path) self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() task.set_state("get_response") return response finally: with self.task_count_lock: self.task_count -= 1 def retry_task_cb(self, task, reason=""): self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() self.logger.warn("retry_task_cb: %s", task.url) if task.responsed: self.logger.warn("retry but responsed. %s", task.url) st = traceback.extract_stack() stl = traceback.format_list(st) self.logger.warn("stack:%r", repr(stl)) task.finish() return if task.retry_count > 10: task.response_fail("retry time exceed 10") return if time.time() - task.start_time > task.timeout: task.response_fail("retry timeout:%d" % (time.time() - task.start_time)) return if not self.running: task.response_fail("retry but stopped.") return task.set_state("retry(%s)" % reason) task.retry_count += 1 self.request_queue.put(task) def dispatcher(self): while self.running: start_time = time.time() try: task = self.request_queue.get(True) if task is None: # exit break except Exception as e: self.logger.exception("http_dispatcher dispatcher request_queue.get fail:%r", e) continue get_time = time.time() get_cost = get_time - start_time task.set_state("get_task(%d)" % get_cost) try: worker = self.get_worker() except Exception as e: self.logger.warn("get worker fail:%r", e) task.response_fail(reason="get worker fail:%r" % e) continue if worker is None: # can send because exit. self.logger.warn("http_dispatcher get None worker") task.response_fail("get worker fail.") continue get_worker_time = time.time() get_cost = get_worker_time - get_time task.set_state("get_worker(%d):%s" % (get_cost, worker.ip)) task.worker = worker try: worker.request(task) except Exception as e: self.logger.exception("dispatch request:%r", e) # wait up threads to exit. self.wait_a_worker_cv.notify() self.trigger_create_worker_cv.notify() def connection_checker(self): while self.running: now = time.time() try: for worker in list(self.workers): if worker.version == "1.1": continue worker.check_active(now) except Exception as e: self.logger.exception("check worker except:%r") time.sleep(1) def is_idle(self): return time.time() - self.last_request_time > self.idle_time def close_cb(self, worker): try: self.workers.remove(worker) if worker.version == "2": self.h2_num -= 1 else: self.h1_num -= 1 except: pass def close_all_worker(self, reason="close all worker"): for w in list(self.workers): if w.accept_task: w.close(reason) self.workers = [] self.h1_num = 0 self.h2_num = 0 def log_debug_data(self, rtt, sent, received): self.rtts.append(rtt) self.total_sent += sent self.total_received += received def statistic(self): now = time.time() if now > self.last_statistic_time + 60: rtt = 0 sent = 0 received = 0 for stat in self.second_stats: rtt = max(rtt, stat["rtt"]) sent += stat["sent"] received += stat["received"] self.minute_stat = { "rtt": rtt, "sent": sent, "received": received } self.second_stats = Queue.deque() self.last_statistic_time = now if len(self.rtts): rtt = max(self.rtts) else: rtt = 0 self.second_stat = { "rtt": rtt, "sent": self.total_sent - self.last_sent, "received": self.total_received - self.last_received } self.rtts = [] self.last_sent = self.total_sent self.last_received = self.total_received self.second_stats.append(self.second_stat) def worker_num(self): return len(self.workers) def get_score(self): now = time.time() if now - self.last_fail_time < 60 and \ self.continue_fail_num > 10: return None worker = self.get_worker(nowait=True) if not worker: return None return worker.get_score() * self.config.dispather_score_factor def to_string(self): now = time.time() worker_rate = {} for w in self.workers: worker_rate[w] = w.get_rtt_rate() w_r = sorted(worker_rate.items(), key=operator.itemgetter(1)) out_str = 'thread num:%d\r\n' % threading.activeCount() for w, r in w_r: out_str += "%s score:%d rtt:%d running:%d accept:%d live:%d inactive:%d processed:%d" % \ (w.ip, w.get_score(), w.rtt, w.keep_running, w.accept_task, (now-w.ssl_sock.create_time), (now-w.last_recv_time), w.processed_tasks) if w.version == "2": out_str += " continue_timeout:%d streams:%d ping_on_way:%d remote_win:%d send_queue:%d\r\n" % \ (w.continue_timeout, len(w.streams), w.ping_on_way, w.remote_window_size, w.send_queue.qsize()) elif w.version == "1.1": out_str += " Trace:%s" % w.get_trace() out_str += "\r\n Speed:" for speed in w.speed_history: out_str += "%d," % speed out_str += "\r\n" out_str += "\r\n working_tasks:\r\n" for unique_id in self.working_tasks: task = self.working_tasks[unique_id] out_str += task.to_string() return out_str
class HttpsDispatcher(object): idle_time = 20 * 60 def __init__(self, host, log_debug_data): self.host = host self.log_debug_data = log_debug_data self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.create_worker_th = None self.last_request_time = time.time() self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = SimpleCondition() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() # move created ssl to worker after ssl timeout self.https_manager = connect_manager.Https_connection_manager( host, self.on_ssl_created_cb) def on_ssl_created_cb(self, ssl_sock, check_free_work=True): if not ssl_sock: raise Exception("on_ssl_created_cb ssl_sock None") if ssl_sock.h2: worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h2_num += 1 else: worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h1_num += 1 self.workers.append(worker) self.wait_a_worker_cv.notify() if check_free_work: self.check_free_worker() def _on_worker_idle_cb(self): self.wait_a_worker_cv.notify() def create_worker_thread(self): while connect_control.keep_running: try: ssl_sock = self.https_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # xlog.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_work=False) except: time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 if idle_num > 1 and acceptable_num > 1: self.triger_create_worker_cv.wait() def get_worker(self, nowait=False): while connect_control.keep_running: best_score = 99999999 best_worker = None idle_num = 0 now = time.time() for worker in self.workers: if not worker.accept_task: continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 score = worker.get_score() if best_score > score: best_score = score best_worker = worker if best_worker is None or idle_num < 5 or ( now - best_worker.last_active_time) < 2: self.triger_create_worker_cv.notify() if best_worker or nowait: return best_worker self.wait_a_worker_cv.wait() def check_free_worker(self): # close slowest worker, # give change for better worker while True: slowest_score = 9999 slowest_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "2" and len(worker.streams) > 0: continue score = worker.get_score() if score < 1000: idle_num += 1 if score > slowest_score: slowest_score = score slowest_worker = worker if idle_num < 10 or idle_num < int(len(self.workers) * 0.3) or len( self.workers) < 50: return if slowest_worker is None: return self.close_cb(slowest_worker) def request(self, method, host, path, headers, body, url="", timeout=60): # xlog.debug("task start request") if not url: url = "%s %s%s" % (method, host, path) self.last_request_time = time.time() q = Queue.Queue() task = http_common.Task(method, host, path, headers, body, q, url, timeout) task.set_state("start_request") self.request_queue.put(task) # self.working_tasks[task.unique_id] = task response = q.get(True) task.set_state("get_response") # del self.working_tasks[task.unique_id] return response def retry_task_cb(self, task): if task.responsed: xlog.warn("retry but responsed. %s", task.url) st = traceback.extract_stack() stl = traceback.format_list(st) xlog.warn("stack:%r", repr(stl)) task.put_data("") return if task.retry_count > 10: task.response_fail("retry time exceed 10") return if time.time() - task.start_time > 240: task.response_fail("retry timeout:%d" % (time.time() - task.start_time)) return task.set_state("retry") task.retry_count += 1 self.request_queue.put(task) def dispatcher(self): while connect_control.keep_running: start_time = time.time() try: task = self.request_queue.get(True) if task is None: # exit break except Exception as e: xlog.exception( "http_dispatcher dispatcher request_queue.get fail:%r", e) continue get_time = time.time() get_cost = get_time - start_time task.set_state("get_task(%d)" % get_cost) try: worker = self.get_worker() except Exception as e: xlog.warn("get worker fail:%r", e) task.response_fail(reason="get worker fail:%r" % e) continue if worker is None: # can send because exit. xlog.warn("http_dispatcher get None worker") task.response_fail("get worker fail.") continue task.set_state("get_worker:%s" % worker.ip) task.worker = worker try: worker.request(task) except Exception as e: xlog.exception("dispatch request:%r", e) # wait up threads to exit. self.wait_a_worker_cv.notify() self.triger_create_worker_cv.notify() def is_idle(self): return time.time() - self.last_request_time > self.idle_time def close_cb(self, worker): try: self.workers.remove(worker) if worker.version == "2": self.h2_num -= 1 else: self.h1_num -= 1 except: pass def close_all_worker(self): for w in self.workers: w.close("close all worker") self.workers = [] self.h1_num = 0 self.h2_num = 0 def to_string(self): worker_rate = {} for w in self.workers: worker_rate[w] = w.get_rtt_rate() w_r = sorted(worker_rate.items(), key=operator.itemgetter(1)) out_str = 'thread num:%d\r\n' % threading.activeCount() for w, r in w_r: out_str += "%s rtt:%d a:%d live:%d processed:%d" % \ (w.ip, w.rtt, w.accept_task, (time.time()-w.ssl_sock.create_time), w.processed_tasks) if w.version == "2": out_str += " streams:%d ping_on_way:%d\r\n" % (len( w.streams), w.ping_on_way) elif w.version == "1.1": out_str += " Trace:%s" % w.get_trace() out_str += "\r\n Speed:" for speed in w.speed_history: out_str += "%d," % speed out_str += "\r\n" out_str += "\r\n working_tasks:\r\n" for unique_id in self.working_tasks: task = self.working_tasks[unique_id] out_str += task.to_string() return out_str
class HttpsDispatcher(object): idle_time = 20 * 60 def __init__(self): self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.create_worker_th = None self.last_request_time = time.time() self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = SimpleCondition() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() # move created ssl to worker after ssl timeout https_manager.set_ssl_time_handler(self.on_ssl_created_cb) def on_ssl_created_cb(self, ssl_sock, check_free_worke=True): if not ssl_sock: raise Exception("on_ssl_created_cb ssl_sock None") appid = appid_manager.get_appid() if not appid: time.sleep(60) ssl_sock.close() raise http_common.GAE_Exception(1, "no appid can use") ssl_sock.appid = appid ssl_sock.host = ssl_sock.appid + ".appspot.com" if ssl_sock.h2: worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h2_num += 1 else: worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h1_num += 1 self.workers.append(worker) self.wait_a_worker_cv.notify() if check_free_worke: self.check_free_worker() def log_debug_data(self, rtt, sent, received): pass def _on_worker_idle_cb(self): self.wait_a_worker_cv.notify() def create_worker_thread(self): while connect_control.keep_running: try: ssl_sock = https_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # xlog.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_worke=False) except: time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 if idle_num > 5 and acceptable_num > 20: self.triger_create_worker_cv.wait() def get_worker(self, nowait=False): while connect_control.keep_running: best_rtt = 9999 best_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 rtt = worker.get_score() if rtt < best_rtt: best_rtt = rtt best_worker = worker if idle_num < 5 or best_rtt > 1000: self.triger_create_worker_cv.notify() if best_worker or nowait: return best_worker self.wait_a_worker_cv.wait() def check_free_worker(self): # close slowest worker, # give change for better worker while True: slowest_rtt = 9999 slowest_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "2" and len(worker.streams) > 0: continue idle_num += 1 rtt = worker.get_score() if rtt > slowest_rtt: slowest_rtt = rtt slowest_worker = worker if idle_num < 30 or idle_num < int(len(self.workers) * 0.3): return if slowest_worker is None: return self.close_cb(slowest_worker) def request(self, headers, body, url, timeout): # xlog.debug("task start request:%s timeout:%d", url, timeout) self.last_request_time = time.time() q = Queue.Queue() task = http_common.Task(headers, body, q, url, timeout) task.set_state("start_request") self.request_queue.put(task) response = q.get(True) task.set_state("get_response") return response def retry_task_cb(self, task): if task.responsed: xlog.warn("retry but responsed. %s", task.url) st = traceback.extract_stack() stl = traceback.format_list(st) xlog.warn("stack:%r", repr(stl)) task.put_data("") return if task.retry_count > 10: task.response_fail("retry time exceed 10") return if time.time() - task.start_time > 240: task.response_fail("retry timeout:%d" % (time.time() - task.start_time)) return task.set_state("retry") task.retry_count += 1 self.request_queue.put(task) def dispatcher(self): while connect_control.keep_running: try: task = self.request_queue.get(True) if task is None: # exit break except Exception as e: xlog.exception("http_dispatcher dispatcher request_queue.get fail:%r", e) continue task.set_state("get_task") try: worker = self.get_worker() except Exception as e: xlog.warn("get worker fail:%r", e) task.response_fail(reason="get worker fail:%r" % e) continue if worker is None: # can send because exit. xlog.warn("http_dispatcher get None worker") continue task.set_state("get_worker:%s" % worker.ip) try: worker.request(task) except Exception as e: xlog.exception("dispatch request:%r", e) # wait up threads to exit. self.wait_a_worker_cv.notify() self.triger_create_worker_cv.notify() def is_idle(self): return time.time() - self.last_request_time > self.idle_time def close_cb(self, worker): try: self.workers.remove(worker) if worker.version == "2": self.h2_num -= 1 else: self.h1_num -= 1 except: pass def close_all_worker(self): for w in self.workers: w.close("close all worker") self.workers = [] self.h1_num = 0 self.h2_num = 0 def to_string(self): worker_rate = {} for w in self.workers: worker_rate[w] = w.get_score() w_r = sorted(worker_rate.items(), key=operator.itemgetter(1)) out_str = 'thread num:%d\r\n' % threading.activeCount() for w, r in w_r: out_str += "%s rtt:%d a:%d live:%d processed:%d" % \ (w.ip, w.rtt, w.accept_task, (time.time()-w.ssl_sock.create_time), w.processed_tasks) if w.version == "2": out_str += " streams:%d ping_on_way:%d\r\n" % (len(w.streams), w.ping_on_way) out_str += " Speed:" for speed in w.speed_history: out_str += "%d," % speed out_str += "\r\n" out_str += "\r\n<br> working_tasks:\r\n" for unique_id in self.working_tasks: task = self.working_tasks[unique_id] out_str += task.to_string() return out_str
class HttpsDispatcher(object): idle_time = 20 * 60 def __init__(self, host, log_debug_data): self.host = host self.log_debug_data = log_debug_data self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.create_worker_th = None self.last_request_time = time.time() self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = SimpleCondition() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() # move created ssl to worker after ssl timeout self.https_manager = connect_manager.Https_connection_manager(host, self.on_ssl_created_cb) def on_ssl_created_cb(self, ssl_sock, check_free_work=True): if not ssl_sock: raise Exception("on_ssl_created_cb ssl_sock None") if ssl_sock.h2: worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h2_num += 1 else: worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h1_num += 1 self.workers.append(worker) self.wait_a_worker_cv.notify() if check_free_work: self.check_free_worker() def _on_worker_idle_cb(self): self.wait_a_worker_cv.notify() def create_worker_thread(self): while connect_control.keep_running: try: ssl_sock = self.https_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # xlog.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_work=False) except: time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 if idle_num > 1 and acceptable_num > 1: self.triger_create_worker_cv.wait() def get_worker(self, nowait=False): while connect_control.keep_running: best_score = 99999999 best_worker = None idle_num = 0 now = time.time() for worker in self.workers: if not worker.accept_task: continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 score = worker.get_score() if best_score > score: best_score = score best_worker = worker if best_worker is None or idle_num < 5 or (now - best_worker.last_active_time) < 2 or best_score>1000: self.triger_create_worker_cv.notify() if nowait: return best_worker if best_worker and (now - best_worker.last_active_time) > 1: return best_worker self.wait_a_worker_cv.wait() def check_free_worker(self): # close slowest worker, # give change for better worker while True: slowest_score = 9999 slowest_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "2" and len(worker.streams) > 0: continue score = worker.get_score() if score < 1000: idle_num += 1 if score > slowest_score: slowest_score = score slowest_worker = worker if idle_num < 10 or idle_num < int(len(self.workers) * 0.3) or len(self.workers) < 50: return if slowest_worker is None: return self.close_cb(slowest_worker) def request(self, method, host, path, headers, body, url="", timeout=60): # xlog.debug("task start request") if not url: url = "%s %s%s" % (method, host, path) self.last_request_time = time.time() q = Queue.Queue() task = http_common.Task(method, host, path, headers, body, q, url, timeout) task.set_state("start_request") self.request_queue.put(task) # self.working_tasks[task.unique_id] = task response = q.get(True) task.set_state("get_response") # del self.working_tasks[task.unique_id] return response def retry_task_cb(self, task, reason=""): if task.responsed: xlog.warn("retry but responsed. %s", task.url) st = traceback.extract_stack() stl = traceback.format_list(st) xlog.warn("stack:%r", repr(stl)) task.put_data("") return if task.retry_count > 10: task.response_fail("retry time exceed 10") return if time.time() - task.start_time > task.timeout: task.response_fail("retry timeout:%d" % (time.time() - task.start_time)) return task.set_state("retry(%s)" % reason) task.retry_count += 1 self.request_queue.put(task) def dispatcher(self): while connect_control.keep_running: start_time = time.time() try: task = self.request_queue.get(True) if task is None: # exit break except Exception as e: xlog.exception("http_dispatcher dispatcher request_queue.get fail:%r", e) continue get_time = time.time() get_cost = get_time - start_time task.set_state("get_task(%d)" % get_cost) try: worker = self.get_worker() except Exception as e: xlog.warn("get worker fail:%r", e) task.response_fail(reason="get worker fail:%r" % e) continue if worker is None: # can send because exit. xlog.warn("http_dispatcher get None worker") task.response_fail("get worker fail.") continue get_worker_time = time.time() get_cost = get_worker_time - get_time task.set_state("get_worker(%d):%s" % (get_cost, worker.ip)) task.worker = worker try: worker.request(task) except Exception as e: xlog.exception("dispatch request:%r", e) # wait up threads to exit. self.wait_a_worker_cv.notify() self.triger_create_worker_cv.notify() def is_idle(self): return time.time() - self.last_request_time > self.idle_time def close_cb(self, worker): try: self.workers.remove(worker) if worker.version == "2": self.h2_num -= 1 else: self.h1_num -= 1 except: pass def close_all_worker(self): for w in self.workers: w.close("close all worker") self.workers = [] self.h1_num = 0 self.h2_num = 0 def to_string(self): now = time.time() worker_rate = {} for w in self.workers: worker_rate[w] = w.get_rtt_rate() w_r = sorted(worker_rate.items(), key=operator.itemgetter(1)) out_str = 'thread num:%d\r\n' % threading.activeCount() for w, r in w_r: out_str += "%s rtt:%d a:%d live:%d inactive:%d processed:%d" % \ (w.ip, w.rtt, w.accept_task, (now-w.ssl_sock.create_time), (now-w.last_active_time), w.processed_tasks) if w.version == "2": out_str += " streams:%d ping_on_way:%d\r\n" % (len(w.streams), w.ping_on_way) elif w.version == "1.1": out_str += " Trace:%s" % w.get_trace() out_str += "\r\n Speed:" for speed in w.speed_history: out_str += "%d," % speed out_str += "\r\n" out_str += "\r\n working_tasks:\r\n" for unique_id in self.working_tasks: task = self.working_tasks[unique_id] out_str += task.to_string() return out_str
class HttpsDispatcher(object): idle_time = 2 * 60 def __init__(self, logger, config, ip_manager, connection_manager, http1worker=Http1Worker, http2worker=Http2Worker): self.logger = logger self.config = config self.ip_manager = ip_manager self.connection_manager = connection_manager self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb) self.http1worker = http1worker self.http2worker = http2worker self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.last_request_time = time.time() self.task_count_lock = threading.Lock() self.task_count = 0 self.running = True # for statistic self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = [] self.last_sent = self.total_sent = 0 self.last_received = self.total_received = 0 self.second_stats = Queue.deque() self.last_statistic_time = time.time() self.second_stat = {"rtt": 0, "sent": 0, "received": 0} self.minute_stat = {"rtt": 0, "sent": 0, "received": 0} self.trigger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = simple_queue.Queue() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() threading.Thread(target=self.connection_checker).start() def stop(self): self.running = False self.request_queue.put(None) self.close_all_worker("stop") def on_ssl_created_cb(self, ssl_sock, check_free_work=True): # self.logger.debug("on_ssl_created_cb %s", ssl_sock.ip) if not self.running: ssl_sock.close() return if not ssl_sock: raise Exception("on_ssl_created_cb ssl_sock None") if ssl_sock.h2: worker = self.http2worker(self.logger, self.ip_manager, self.config, ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h2_num += 1 else: worker = self.http1worker(self.logger, self.ip_manager, self.config, ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data) self.h1_num += 1 self.workers.append(worker) if check_free_work: self.check_free_worker() def _on_worker_idle_cb(self): self.wait_a_worker_cv.notify() def create_worker_thread(self): while self.running: self.trigger_create_worker_cv.wait() try: ssl_sock = self.connection_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # self.logger.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_work=False) except Exception as e: #self.logger.exception("on_ssl_created_cb except:%r", e) time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 def get_worker(self, nowait=False): while self.running: best_score = 99999999 best_worker = None idle_num = 0 now = time.time() for worker in self.workers: if not worker.accept_task: # self.logger.debug("not accept") continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 score = worker.get_score() if best_score > score: best_score = score best_worker = worker if len(self.workers) < self.config.dispather_max_workers and \ (best_worker is None or idle_num < self.config.dispather_min_idle_workers or len(self.workers) < self.config.dispather_min_workers or (now - best_worker.last_recv_time) < self.config.dispather_work_min_idle_time or best_score > self.config.dispather_work_max_score or (best_worker.version == "2" and len(best_worker.streams) >= self.config.http2_target_concurrent)): # self.logger.debug("trigger get more worker") self.trigger_create_worker_cv.notify() if nowait or \ (best_worker and (now - best_worker.last_recv_time) >= self.config.dispather_work_min_idle_time): # self.logger.debug("return worker") return best_worker self.wait_a_worker_cv.wait(time.time() + 1) # self.logger.debug("get wait_a_worker_cv") #time.sleep(0.1) def check_free_worker(self): # close slowest worker, # give change for better worker while True: slowest_score = 9999 slowest_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "2" and len(worker.streams) > 0: continue score = worker.get_score() if score < 1000: idle_num += 1 if score > slowest_score: slowest_score = score slowest_worker = worker if idle_num < self.config.dispather_max_idle_workers or \ idle_num < int(len(self.workers) * 0.3) or \ len(self.workers) < self.config.dispather_max_workers: return if slowest_worker is None: return self.close_cb(slowest_worker) def request(self, method, host, path, headers, body, url="", timeout=60): if self.task_count > self.config.max_task_num: self.logger.warn("task num exceed") time.sleep(1) return None with self.task_count_lock: self.task_count += 1 try: # self.logger.debug("task start request") if not url: url = "%s %s%s" % (method, host, path) self.last_request_time = time.time() q = simple_queue.Queue() task = http_common.Task(self.logger, self.config, method, host, path, headers, body, q, url, timeout) task.set_state("start_request") self.request_queue.put(task) response = q.get(timeout=timeout) if response and response.status == 200: self.success_num += 1 self.continue_fail_num = 0 else: self.logger.warn("task %s %s %s timeout", method, host, path) self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() task.set_state("get_response") return response finally: with self.task_count_lock: self.task_count -= 1 def retry_task_cb(self, task, reason=""): self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() self.logger.warn("retry_task_cb: %s", task.url) if task.responsed: self.logger.warn("retry but responsed. %s", task.url) st = traceback.extract_stack() stl = traceback.format_list(st) self.logger.warn("stack:%r", repr(stl)) task.finish() return if task.retry_count > 10: task.response_fail("retry time exceed 10") return if time.time() - task.start_time > task.timeout: task.response_fail("retry timeout:%d" % (time.time() - task.start_time)) return if not self.running: task.response_fail("retry but stopped.") return task.set_state("retry(%s)" % reason) task.retry_count += 1 self.request_queue.put(task) def dispatcher(self): while self.running: start_time = time.time() try: task = self.request_queue.get(True) if task is None: # exit break except Exception as e: self.logger.exception( "http_dispatcher dispatcher request_queue.get fail:%r", e) continue get_time = time.time() get_cost = get_time - start_time task.set_state("get_task(%d)" % get_cost) try: worker = self.get_worker() except Exception as e: self.logger.warn("get worker fail:%r", e) task.response_fail(reason="get worker fail:%r" % e) continue if worker is None: # can send because exit. self.logger.warn("http_dispatcher get None worker") task.response_fail("get worker fail.") continue get_worker_time = time.time() get_cost = get_worker_time - get_time task.set_state("get_worker(%d):%s" % (get_cost, worker.ip_str)) task.worker = worker try: worker.request(task) except Exception as e: self.logger.exception("dispatch request:%r", e) # wait up threads to exit. self.wait_a_worker_cv.notify() self.trigger_create_worker_cv.notify() def connection_checker(self): while self.running: now = time.time() try: for worker in list(self.workers): if worker.version == "1.1": continue worker.check_active(now) except Exception as e: self.logger.exception("check worker except:%r") time.sleep(1) def is_idle(self): return time.time() - self.last_request_time > self.idle_time def close_cb(self, worker): try: self.workers.remove(worker) if worker.version == "2": self.h2_num -= 1 else: self.h1_num -= 1 except: pass def close_all_worker(self, reason="close all worker"): for w in list(self.workers): if w.accept_task: w.close(reason) self.workers = [] self.h1_num = 0 self.h2_num = 0 def log_debug_data(self, rtt, sent, received): self.rtts.append(rtt) self.total_sent += sent self.total_received += received def statistic(self): now = time.time() if now > self.last_statistic_time + 60: rtt = 0 sent = 0 received = 0 for stat in self.second_stats: rtt = max(rtt, stat["rtt"]) sent += stat["sent"] received += stat["received"] self.minute_stat = {"rtt": rtt, "sent": sent, "received": received} self.second_stats = Queue.deque() self.last_statistic_time = now if len(self.rtts): rtt = max(self.rtts) else: rtt = 0 self.second_stat = { "rtt": rtt, "sent": self.total_sent - self.last_sent, "received": self.total_received - self.last_received } self.rtts = [] self.last_sent = self.total_sent self.last_received = self.total_received self.second_stats.append(self.second_stat) def worker_num(self): return len(self.workers) def get_score(self): now = time.time() if now - self.last_fail_time < 60 and \ self.continue_fail_num > 10: return None worker = self.get_worker(nowait=True) if not worker: return None return worker.get_score() * self.config.dispather_score_factor def to_string(self): now = time.time() worker_rate = {} for w in self.workers: worker_rate[w] = w.get_rtt_rate() w_r = sorted(worker_rate.items(), key=operator.itemgetter(1)) out_str = 'thread num:%d\r\n' % threading.activeCount() for w, r in w_r: out_str += "%s score:%d rtt:%d running:%d accept:%d live:%d inactive:%d processed:%d" % \ (w.ip_str, w.get_score(), w.rtt, w.keep_running, w.accept_task, (now-w.ssl_sock.create_time), (now-w.last_recv_time), w.processed_tasks) if w.version == "2": out_str += " continue_timeout:%d streams:%d ping_on_way:%d remote_win:%d send_queue:%d\r\n" % \ (w.continue_timeout, len(w.streams), w.ping_on_way, w.remote_window_size, w.send_queue.qsize()) elif w.version == "1.1": out_str += " Trace:%s" % w.get_trace() out_str += "\r\n Speed:" for speed in w.speed_history: out_str += "%d," % speed out_str += "\r\n" out_str += "\r\n working_tasks:\r\n" for unique_id in self.working_tasks: task = self.working_tasks[unique_id] out_str += task.to_string() return out_str
class HttpsDispatcher(object): idle_time = 20 * 60 def __init__(self): self.request_queue = Queue.Queue() self.workers = [] self.working_tasks = {} self.h1_num = 0 self.h2_num = 0 self.create_worker_th = None self.last_request_time = time.time() self.triger_create_worker_cv = SimpleCondition() self.wait_a_worker_cv = SimpleCondition() threading.Thread(target=self.dispatcher).start() threading.Thread(target=self.create_worker_thread).start() # move created ssl to worker after ssl timeout https_manager.set_ssl_time_handler(self.on_ssl_created_cb) def on_ssl_created_cb(self, ssl_sock, check_free_worke=True): if not ssl_sock: raise Exception("on_ssl_created_cb ssl_sock None") appid = appid_manager.get_appid() if not appid: time.sleep(60) ssl_sock.close() raise http_common.GAE_Exception(1, "no appid can use") ssl_sock.appid = appid ssl_sock.host = ssl_sock.appid + ".appspot.com" if ssl_sock.h2: worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb) self.h2_num += 1 else: worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb) self.h1_num += 1 self.workers.append(worker) self.wait_a_worker_cv.notify() if check_free_worke: self.check_free_worker() def _on_worker_idle_cb(self): self.wait_a_worker_cv.notify() def create_worker_thread(self): while connect_control.keep_running: try: ssl_sock = https_manager.get_ssl_connection() except Exception as e: continue if not ssl_sock: # xlog.warn("create_worker_thread get ssl_sock fail") continue try: self.on_ssl_created_cb(ssl_sock, check_free_worke=False) except: time.sleep(10) idle_num = 0 acceptable_num = 0 for worker in self.workers: if worker.accept_task: acceptable_num += 1 if worker.version == "1.1": if worker.accept_task: idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 if idle_num > 5 and acceptable_num > 20: self.triger_create_worker_cv.wait() def get_worker(self): while connect_control.keep_running: best_rtt = 9999 best_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "1.1": idle_num += 1 else: if len(worker.streams) == 0: idle_num += 1 rtt = worker.get_rtt_rate() if rtt < best_rtt: best_rtt = rtt best_worker = worker if idle_num < 5: self.triger_create_worker_cv.notify() if best_worker: return best_worker self.wait_a_worker_cv.wait() def check_free_worker(self): # close slowest worker, # give change for better worker while True: slowest_rtt = 9999 slowest_worker = None idle_num = 0 for worker in self.workers: if not worker.accept_task: continue if worker.version == "2" and len(worker.streams) > 0: continue idle_num += 1 rtt = worker.get_rtt_rate() if rtt > slowest_rtt: slowest_rtt = rtt slowest_worker = worker if idle_num < 30 or idle_num < int(len(self.workers) * 0.3): return if slowest_worker is None: return self.close_cb(slowest_worker) def request(self, headers, body, url): # xlog.debug("task start request") self.last_request_time = time.time() q = Queue.Queue() task = http_common.Task(headers, body, q, url) unique_id = task.unique_id task.set_state("start_request") self.request_queue.put(task) self.working_tasks[task.unique_id] = task response = q.get(True) task.set_state("get_response") try: del self.working_tasks[task.unique_id] except Exception as e: xlog.error("http_dispatcher request unique_id %s, %s not found.", unique_id, task.unique_id) return response def retry_task_cb(self, task): if task.responsed: xlog.warn("retry but responsed. %s", task.url) st = traceback.extract_stack() stl = traceback.format_list(st) xlog.warn("stack:%r", repr(stl)) task.put_data("") return if task.retry_count > 10: task.response_fail("retry time exceed 10") return if time.time() - task.start_time > 240: task.response_fail("retry timeout:%d" % (time.time() - task.start_time)) return task.set_state("retry") task.retry_count += 1 self.request_queue.put(task) def dispatcher(self): while connect_control.keep_running: try: task = self.request_queue.get(True) if task is None: # exit break except Exception as e: xlog.exception("http_dispatcher dispatcher request_queue.get fail:%r", e) continue task.set_state("get_task") try: worker = self.get_worker() except Exception as e: xlog.warn("get worker fail:%r", e) task.response_fail(reason="get worker fail:%r" % e) continue if worker is None: # can send because exit. xlog.warn("http_dispatcher get None worker") continue task.set_state("get_worker:%s" % worker.ip) try: worker.request(task) except Exception as e: xlog.exception("dispatch request:%r", e) # wait up threads to exit. self.wait_a_worker_cv.notify() self.triger_create_worker_cv.notify() def is_idle(self): return time.time() - self.last_request_time > self.idle_time def close_cb(self, worker): try: self.workers.remove(worker) if worker.version == "2": self.h2_num -= 1 else: self.h1_num -= 1 except: pass def close_all_worker(self): for w in self.workers: w.close("close all worker") self.workers = [] self.h1_num = 0 self.h2_num = 0 def to_string(self): worker_rate = {} for w in self.workers: worker_rate[w] = w.get_rtt_rate() w_r = sorted(worker_rate.items(), key=operator.itemgetter(1)) out_str = 'thread num:%d\r\n' % threading.activeCount() for w, r in w_r: out_str += "%s rtt:%d a:%d live:%d processed:%d" % \ (w.ip, w.rtt, w.accept_task, (time.time()-w.ssl_sock.create_time), w.processed_tasks) if w.version == "2": out_str += " streams:%d ping_on_way:%d\r\n" % (len(w.streams), w.ping_on_way) out_str += " Speed:" for speed in w.speed_history: out_str += "%d," % speed out_str += "\r\n" out_str += "\r\n<br> working_tasks:\r\n" for unique_id in self.working_tasks: task = self.working_tasks[unique_id] out_str += task.to_string() return out_str