def __init__(self,
logger,
config,
ip_manager,
connection_manager,
http1worker=Http1Worker,
http2worker=Http2Worker):
self.logger = logger
self.config = config
self.ip_manager = ip_manager
self.connection_manager = connection_manager
self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb)
self.http1worker = http1worker
self.http2worker = http2worker
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.last_request_time = time.time()
self.running = True
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = simple_queue.Queue()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
def __init__(self, logger, config, ip_manager, connection_manager,
http1worker=Http1Worker,
http2worker=Http2Worker):
self.logger = logger
self.config = config
self.ip_manager = ip_manager
self.connection_manager = connection_manager
self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb)
self.http1worker = http1worker
self.http2worker = http2worker
self.request_queue = queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.last_request_time = time.time()
self.task_count_lock = threading.Lock()
self.task_count = 0
self.running = True
# for statistic
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.rtts = []
self.last_sent = self.total_sent = 0
self.last_received = self.total_received = 0
self.second_stats = queue.deque()
self.last_statistic_time = time.time()
self.second_stat = {
"rtt": 0,
"sent": 0,
"received": 0
}
self.minute_stat = {
"rtt": 0,
"sent": 0,
"received": 0
}
self.trigger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = simple_queue.Queue()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
threading.Thread(target=self.connection_checker).start()
def __init__(self, logger, config, ip_manager, connection_manager,
http1worker=Http1Worker,
http2worker=Http2Worker):
self.logger = logger
self.config = config
self.ip_manager = ip_manager
self.connection_manager = connection_manager
self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb)
self.http1worker = http1worker
self.http2worker = http2worker
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.last_request_time = time.time()
self.running = True
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = simple_queue.Queue()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
def __init__(self):
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
https_manager.set_ssl_time_handler(self.on_ssl_created_cb)
def __init__(self):
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
https_manager.set_ssl_time_handler(self.on_ssl_created_cb)
def __init__(self, host, log_debug_data):
self.host = host
self.log_debug_data = log_debug_data
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
self.https_manager = connect_manager.Https_connection_manager(host, self.on_ssl_created_cb)
def __init__(self, host, log_debug_data):
self.host = host
self.log_debug_data = log_debug_data
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
self.https_manager = connect_manager.Https_connection_manager(host, self.on_ssl_created_cb)
def __init__(self, logger, config, ip_manager, connection_manager,
http1worker=Http1Worker,
http2worker=Http2Worker):
self.logger = logger
self.config = config
self.ip_manager = ip_manager
self.connection_manager = connection_manager
self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb)
self.http1worker = http1worker
self.http2worker = http2worker
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.last_request_time = time.time()
self.task_count_lock = threading.Lock()
self.task_count = 0
self.running = True
# for statistic
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.rtts = []
self.last_sent = self.total_sent = 0
self.last_received = self.total_received = 0
self.second_stats = Queue.deque()
self.last_statistic_time = time.time()
self.second_stat = {
"rtt": 0,
"sent": 0,
"received": 0
}
self.minute_stat = {
"rtt": 0,
"sent": 0,
"received": 0
}
self.trigger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = simple_queue.Queue()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
threading.Thread(target=self.connection_checker).start()
class HttpsDispatcher(object):
idle_time = 2 * 60
def __init__(self, logger, config, ip_manager, connection_manager,
http1worker=Http1Worker,
http2worker=Http2Worker):
self.logger = logger
self.config = config
self.ip_manager = ip_manager
self.connection_manager = connection_manager
self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb)
self.http1worker = http1worker
self.http2worker = http2worker
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.last_request_time = time.time()
self.task_count_lock = threading.Lock()
self.task_count = 0
self.running = True
# for statistic
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.rtts = []
self.last_sent = self.total_sent = 0
self.last_received = self.total_received = 0
self.second_stats = Queue.deque()
self.last_statistic_time = time.time()
self.second_stat = {
"rtt": 0,
"sent": 0,
"received": 0
}
self.minute_stat = {
"rtt": 0,
"sent": 0,
"received": 0
}
self.trigger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = simple_queue.Queue()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
threading.Thread(target=self.connection_checker).start()
def stop(self):
self.running = False
self.request_queue.put(None)
self.close_all_worker("stop")
def on_ssl_created_cb(self, ssl_sock, check_free_work=True):
# self.logger.debug("on_ssl_created_cb %s", ssl_sock.ip)
if not self.running:
ssl_sock.close()
return
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
if ssl_sock.h2:
worker = self.http2worker(
self.logger, self.ip_manager, self.config, ssl_sock,
self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h2_num += 1
else:
worker = self.http1worker(
self.logger, self.ip_manager, self.config, ssl_sock,
self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h1_num += 1
self.workers.append(worker)
if check_free_work:
self.check_free_worker()
def _on_worker_idle_cb(self):
self.wait_a_worker_cv.notify()
def create_worker_thread(self):
while self.running:
self.trigger_create_worker_cv.wait()
try:
ssl_sock = self.connection_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# self.logger.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_work=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
def get_worker(self, nowait=False):
while self.running:
best_score = 99999999
best_worker = None
idle_num = 0
now = time.time()
for worker in self.workers:
if not worker.accept_task:
# self.logger.debug("not accept")
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
score = worker.get_score()
if best_score > score:
best_score = score
best_worker = worker
if len(self.workers) < self.config.dispather_max_workers and \
(best_worker is None or
idle_num < self.config.dispather_min_idle_workers or
len(self.workers) < self.config.dispather_min_workers or
(now - best_worker.last_recv_time) < self.config.dispather_work_min_idle_time or
best_score > self.config.dispather_work_max_score or
(best_worker.version == "2" and len(best_worker.streams) >= self.config.http2_target_concurrent)):
# self.logger.debug("trigger get more worker")
self.trigger_create_worker_cv.notify()
if nowait or \
(best_worker and (now - best_worker.last_recv_time) >= self.config.dispather_work_min_idle_time):
# self.logger.debug("return worker")
return best_worker
self.wait_a_worker_cv.wait(time.time() + 1)
# self.logger.debug("get wait_a_worker_cv")
#time.sleep(0.1)
def check_free_worker(self):
# close slowest worker,
# give change for better worker
while True:
slowest_score = 9999
slowest_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "2" and len(worker.streams) > 0:
continue
score = worker.get_score()
if score < 1000:
idle_num += 1
if score > slowest_score:
slowest_score = score
slowest_worker = worker
if idle_num < self.config.dispather_max_idle_workers or \
idle_num < int(len(self.workers) * 0.3) or \
len(self.workers) < self.config.dispather_max_workers:
return
if slowest_worker is None:
return
self.close_cb(slowest_worker)
def request(self, method, host, path, headers, body, url="", timeout=60):
if self.task_count > self.config.max_task_num:
self.logger.warn("task num exceed")
time.sleep(1)
return None
with self.task_count_lock:
self.task_count += 1
try:
# self.logger.debug("task start request")
if not url:
url = "%s %s%s" % (method, host, path)
self.last_request_time = time.time()
q = simple_queue.Queue()
task = http_common.Task(self.logger, self.config, method, host, path, headers, body, q, url, timeout)
task.set_state("start_request")
self.request_queue.put(task)
response = q.get(timeout=timeout)
if response and response.status == 200:
self.success_num += 1
self.continue_fail_num = 0
else:
self.logger.warn("task %s %s %s timeout", method, host, path)
self.fail_num += 1
self.continue_fail_num += 1
self.last_fail_time = time.time()
task.set_state("get_response")
return response
finally:
with self.task_count_lock:
self.task_count -= 1
def retry_task_cb(self, task, reason=""):
self.fail_num += 1
self.continue_fail_num += 1
self.last_fail_time = time.time()
self.logger.warn("retry_task_cb: %s", task.url)
if task.responsed:
self.logger.warn("retry but responsed. %s", task.url)
st = traceback.extract_stack()
stl = traceback.format_list(st)
self.logger.warn("stack:%r", repr(stl))
task.finish()
return
if task.retry_count > 10:
task.response_fail("retry time exceed 10")
return
if time.time() - task.start_time > task.timeout:
task.response_fail("retry timeout:%d" % (time.time() - task.start_time))
return
if not self.running:
task.response_fail("retry but stopped.")
return
task.set_state("retry(%s)" % reason)
task.retry_count += 1
self.request_queue.put(task)
def dispatcher(self):
while self.running:
start_time = time.time()
try:
task = self.request_queue.get(True)
if task is None:
# exit
break
except Exception as e:
self.logger.exception("http_dispatcher dispatcher request_queue.get fail:%r", e)
continue
get_time = time.time()
get_cost = get_time - start_time
task.set_state("get_task(%d)" % get_cost)
try:
worker = self.get_worker()
except Exception as e:
self.logger.warn("get worker fail:%r", e)
task.response_fail(reason="get worker fail:%r" % e)
continue
if worker is None:
# can send because exit.
self.logger.warn("http_dispatcher get None worker")
task.response_fail("get worker fail.")
continue
get_worker_time = time.time()
get_cost = get_worker_time - get_time
task.set_state("get_worker(%d):%s" % (get_cost, worker.ip))
task.worker = worker
try:
worker.request(task)
except Exception as e:
self.logger.exception("dispatch request:%r", e)
# wait up threads to exit.
self.wait_a_worker_cv.notify()
self.trigger_create_worker_cv.notify()
def connection_checker(self):
while self.running:
now = time.time()
try:
for worker in list(self.workers):
if worker.version == "1.1":
continue
worker.check_active(now)
except Exception as e:
self.logger.exception("check worker except:%r")
time.sleep(1)
def is_idle(self):
return time.time() - self.last_request_time > self.idle_time
def close_cb(self, worker):
try:
self.workers.remove(worker)
if worker.version == "2":
self.h2_num -= 1
else:
self.h1_num -= 1
except:
pass
def close_all_worker(self, reason="close all worker"):
for w in list(self.workers):
if w.accept_task:
w.close(reason)
self.workers = []
self.h1_num = 0
self.h2_num = 0
def log_debug_data(self, rtt, sent, received):
self.rtts.append(rtt)
self.total_sent += sent
self.total_received += received
def statistic(self):
now = time.time()
if now > self.last_statistic_time + 60:
rtt = 0
sent = 0
received = 0
for stat in self.second_stats:
rtt = max(rtt, stat["rtt"])
sent += stat["sent"]
received += stat["received"]
self.minute_stat = {
"rtt": rtt,
"sent": sent,
"received": received
}
self.second_stats = Queue.deque()
self.last_statistic_time = now
if len(self.rtts):
rtt = max(self.rtts)
else:
rtt = 0
self.second_stat = {
"rtt": rtt,
"sent": self.total_sent - self.last_sent,
"received": self.total_received - self.last_received
}
self.rtts = []
self.last_sent = self.total_sent
self.last_received = self.total_received
self.second_stats.append(self.second_stat)
def worker_num(self):
return len(self.workers)
def get_score(self):
now = time.time()
if now - self.last_fail_time < 60 and \
self.continue_fail_num > 10:
return None
worker = self.get_worker(nowait=True)
if not worker:
return None
return worker.get_score() * self.config.dispather_score_factor
def to_string(self):
now = time.time()
worker_rate = {}
for w in self.workers:
worker_rate[w] = w.get_rtt_rate()
w_r = sorted(worker_rate.items(), key=operator.itemgetter(1))
out_str = 'thread num:%d\r\n' % threading.activeCount()
for w, r in w_r:
out_str += "%s score:%d rtt:%d running:%d accept:%d live:%d inactive:%d processed:%d" % \
(w.ip, w.get_score(), w.rtt, w.keep_running, w.accept_task,
(now-w.ssl_sock.create_time), (now-w.last_recv_time), w.processed_tasks)
if w.version == "2":
out_str += " continue_timeout:%d streams:%d ping_on_way:%d remote_win:%d send_queue:%d\r\n" % \
(w.continue_timeout, len(w.streams), w.ping_on_way, w.remote_window_size, w.send_queue.qsize())
elif w.version == "1.1":
out_str += " Trace:%s" % w.get_trace()
out_str += "\r\n Speed:"
for speed in w.speed_history:
out_str += "%d," % speed
out_str += "\r\n"
out_str += "\r\n working_tasks:\r\n"
for unique_id in self.working_tasks:
task = self.working_tasks[unique_id]
out_str += task.to_string()
return out_str
class HttpsDispatcher(object):
idle_time = 20 * 60
def __init__(self, host, log_debug_data):
self.host = host
self.log_debug_data = log_debug_data
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
self.https_manager = connect_manager.Https_connection_manager(
host, self.on_ssl_created_cb)
def on_ssl_created_cb(self, ssl_sock, check_free_work=True):
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb,
self._on_worker_idle_cb, self.log_debug_data)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb,
self._on_worker_idle_cb, self.log_debug_data)
self.h1_num += 1
self.workers.append(worker)
self.wait_a_worker_cv.notify()
if check_free_work:
self.check_free_worker()
def _on_worker_idle_cb(self):
self.wait_a_worker_cv.notify()
def create_worker_thread(self):
while connect_control.keep_running:
try:
ssl_sock = self.https_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# xlog.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_work=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
if idle_num > 1 and acceptable_num > 1:
self.triger_create_worker_cv.wait()
def get_worker(self, nowait=False):
while connect_control.keep_running:
best_score = 99999999
best_worker = None
idle_num = 0
now = time.time()
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
score = worker.get_score()
if best_score > score:
best_score = score
best_worker = worker
if best_worker is None or idle_num < 5 or (
now - best_worker.last_active_time) < 2:
self.triger_create_worker_cv.notify()
if best_worker or nowait:
return best_worker
self.wait_a_worker_cv.wait()
def check_free_worker(self):
# close slowest worker,
# give change for better worker
while True:
slowest_score = 9999
slowest_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "2" and len(worker.streams) > 0:
continue
score = worker.get_score()
if score < 1000:
idle_num += 1
if score > slowest_score:
slowest_score = score
slowest_worker = worker
if idle_num < 10 or idle_num < int(len(self.workers) * 0.3) or len(
self.workers) < 50:
return
if slowest_worker is None:
return
self.close_cb(slowest_worker)
def request(self, method, host, path, headers, body, url="", timeout=60):
# xlog.debug("task start request")
if not url:
url = "%s %s%s" % (method, host, path)
self.last_request_time = time.time()
q = Queue.Queue()
task = http_common.Task(method, host, path, headers, body, q, url,
timeout)
task.set_state("start_request")
self.request_queue.put(task)
# self.working_tasks[task.unique_id] = task
response = q.get(True)
task.set_state("get_response")
# del self.working_tasks[task.unique_id]
return response
def retry_task_cb(self, task):
if task.responsed:
xlog.warn("retry but responsed. %s", task.url)
st = traceback.extract_stack()
stl = traceback.format_list(st)
xlog.warn("stack:%r", repr(stl))
task.put_data("")
return
if task.retry_count > 10:
task.response_fail("retry time exceed 10")
return
if time.time() - task.start_time > 240:
task.response_fail("retry timeout:%d" %
(time.time() - task.start_time))
return
task.set_state("retry")
task.retry_count += 1
self.request_queue.put(task)
def dispatcher(self):
while connect_control.keep_running:
start_time = time.time()
try:
task = self.request_queue.get(True)
if task is None:
# exit
break
except Exception as e:
xlog.exception(
"http_dispatcher dispatcher request_queue.get fail:%r", e)
continue
get_time = time.time()
get_cost = get_time - start_time
task.set_state("get_task(%d)" % get_cost)
try:
worker = self.get_worker()
except Exception as e:
xlog.warn("get worker fail:%r", e)
task.response_fail(reason="get worker fail:%r" % e)
continue
if worker is None:
# can send because exit.
xlog.warn("http_dispatcher get None worker")
task.response_fail("get worker fail.")
continue
task.set_state("get_worker:%s" % worker.ip)
task.worker = worker
try:
worker.request(task)
except Exception as e:
xlog.exception("dispatch request:%r", e)
# wait up threads to exit.
self.wait_a_worker_cv.notify()
self.triger_create_worker_cv.notify()
def is_idle(self):
return time.time() - self.last_request_time > self.idle_time
def close_cb(self, worker):
try:
self.workers.remove(worker)
if worker.version == "2":
self.h2_num -= 1
else:
self.h1_num -= 1
except:
pass
def close_all_worker(self):
for w in self.workers:
w.close("close all worker")
self.workers = []
self.h1_num = 0
self.h2_num = 0
def to_string(self):
worker_rate = {}
for w in self.workers:
worker_rate[w] = w.get_rtt_rate()
w_r = sorted(worker_rate.items(), key=operator.itemgetter(1))
out_str = 'thread num:%d\r\n' % threading.activeCount()
for w, r in w_r:
out_str += "%s rtt:%d a:%d live:%d processed:%d" % \
(w.ip, w.rtt, w.accept_task, (time.time()-w.ssl_sock.create_time), w.processed_tasks)
if w.version == "2":
out_str += " streams:%d ping_on_way:%d\r\n" % (len(
w.streams), w.ping_on_way)
elif w.version == "1.1":
out_str += " Trace:%s" % w.get_trace()
out_str += "\r\n Speed:"
for speed in w.speed_history:
out_str += "%d," % speed
out_str += "\r\n"
out_str += "\r\n working_tasks:\r\n"
for unique_id in self.working_tasks:
task = self.working_tasks[unique_id]
out_str += task.to_string()
return out_str
class HttpsDispatcher(object):
idle_time = 20 * 60
def __init__(self):
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
https_manager.set_ssl_time_handler(self.on_ssl_created_cb)
def on_ssl_created_cb(self, ssl_sock, check_free_worke=True):
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
appid = appid_manager.get_appid()
if not appid:
time.sleep(60)
ssl_sock.close()
raise http_common.GAE_Exception(1, "no appid can use")
ssl_sock.appid = appid
ssl_sock.host = ssl_sock.appid + ".appspot.com"
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h1_num += 1
self.workers.append(worker)
self.wait_a_worker_cv.notify()
if check_free_worke:
self.check_free_worker()
def log_debug_data(self, rtt, sent, received):
pass
def _on_worker_idle_cb(self):
self.wait_a_worker_cv.notify()
def create_worker_thread(self):
while connect_control.keep_running:
try:
ssl_sock = https_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# xlog.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_worke=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
if idle_num > 5 and acceptable_num > 20:
self.triger_create_worker_cv.wait()
def get_worker(self, nowait=False):
while connect_control.keep_running:
best_rtt = 9999
best_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
rtt = worker.get_score()
if rtt < best_rtt:
best_rtt = rtt
best_worker = worker
if idle_num < 5 or best_rtt > 1000:
self.triger_create_worker_cv.notify()
if best_worker or nowait:
return best_worker
self.wait_a_worker_cv.wait()
def check_free_worker(self):
# close slowest worker,
# give change for better worker
while True:
slowest_rtt = 9999
slowest_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "2" and len(worker.streams) > 0:
continue
idle_num += 1
rtt = worker.get_score()
if rtt > slowest_rtt:
slowest_rtt = rtt
slowest_worker = worker
if idle_num < 30 or idle_num < int(len(self.workers) * 0.3):
return
if slowest_worker is None:
return
self.close_cb(slowest_worker)
def request(self, headers, body, url, timeout):
# xlog.debug("task start request:%s timeout:%d", url, timeout)
self.last_request_time = time.time()
q = Queue.Queue()
task = http_common.Task(headers, body, q, url, timeout)
task.set_state("start_request")
self.request_queue.put(task)
response = q.get(True)
task.set_state("get_response")
return response
def retry_task_cb(self, task):
if task.responsed:
xlog.warn("retry but responsed. %s", task.url)
st = traceback.extract_stack()
stl = traceback.format_list(st)
xlog.warn("stack:%r", repr(stl))
task.put_data("")
return
if task.retry_count > 10:
task.response_fail("retry time exceed 10")
return
if time.time() - task.start_time > 240:
task.response_fail("retry timeout:%d" % (time.time() - task.start_time))
return
task.set_state("retry")
task.retry_count += 1
self.request_queue.put(task)
def dispatcher(self):
while connect_control.keep_running:
try:
task = self.request_queue.get(True)
if task is None:
# exit
break
except Exception as e:
xlog.exception("http_dispatcher dispatcher request_queue.get fail:%r", e)
continue
task.set_state("get_task")
try:
worker = self.get_worker()
except Exception as e:
xlog.warn("get worker fail:%r", e)
task.response_fail(reason="get worker fail:%r" % e)
continue
if worker is None:
# can send because exit.
xlog.warn("http_dispatcher get None worker")
continue
task.set_state("get_worker:%s" % worker.ip)
try:
worker.request(task)
except Exception as e:
xlog.exception("dispatch request:%r", e)
# wait up threads to exit.
self.wait_a_worker_cv.notify()
self.triger_create_worker_cv.notify()
def is_idle(self):
return time.time() - self.last_request_time > self.idle_time
def close_cb(self, worker):
try:
self.workers.remove(worker)
if worker.version == "2":
self.h2_num -= 1
else:
self.h1_num -= 1
except:
pass
def close_all_worker(self):
for w in self.workers:
w.close("close all worker")
self.workers = []
self.h1_num = 0
self.h2_num = 0
def to_string(self):
worker_rate = {}
for w in self.workers:
worker_rate[w] = w.get_score()
w_r = sorted(worker_rate.items(), key=operator.itemgetter(1))
out_str = 'thread num:%d\r\n' % threading.activeCount()
for w, r in w_r:
out_str += "%s rtt:%d a:%d live:%d processed:%d" % \
(w.ip, w.rtt, w.accept_task, (time.time()-w.ssl_sock.create_time), w.processed_tasks)
if w.version == "2":
out_str += " streams:%d ping_on_way:%d\r\n" % (len(w.streams), w.ping_on_way)
out_str += " Speed:"
for speed in w.speed_history:
out_str += "%d," % speed
out_str += "\r\n"
out_str += "\r\n<br> working_tasks:\r\n"
for unique_id in self.working_tasks:
task = self.working_tasks[unique_id]
out_str += task.to_string()
return out_str
class HttpsDispatcher(object):
idle_time = 20 * 60
def __init__(self, host, log_debug_data):
self.host = host
self.log_debug_data = log_debug_data
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
self.https_manager = connect_manager.Https_connection_manager(host, self.on_ssl_created_cb)
def on_ssl_created_cb(self, ssl_sock, check_free_work=True):
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h1_num += 1
self.workers.append(worker)
self.wait_a_worker_cv.notify()
if check_free_work:
self.check_free_worker()
def _on_worker_idle_cb(self):
self.wait_a_worker_cv.notify()
def create_worker_thread(self):
while connect_control.keep_running:
try:
ssl_sock = self.https_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# xlog.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_work=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
if idle_num > 1 and acceptable_num > 1:
self.triger_create_worker_cv.wait()
def get_worker(self, nowait=False):
while connect_control.keep_running:
best_score = 99999999
best_worker = None
idle_num = 0
now = time.time()
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
score = worker.get_score()
if best_score > score:
best_score = score
best_worker = worker
if best_worker is None or idle_num < 5 or (now - best_worker.last_active_time) < 2 or best_score>1000:
self.triger_create_worker_cv.notify()
if nowait:
return best_worker
if best_worker and (now - best_worker.last_active_time) > 1:
return best_worker
self.wait_a_worker_cv.wait()
def check_free_worker(self):
# close slowest worker,
# give change for better worker
while True:
slowest_score = 9999
slowest_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "2" and len(worker.streams) > 0:
continue
score = worker.get_score()
if score < 1000:
idle_num += 1
if score > slowest_score:
slowest_score = score
slowest_worker = worker
if idle_num < 10 or idle_num < int(len(self.workers) * 0.3) or len(self.workers) < 50:
return
if slowest_worker is None:
return
self.close_cb(slowest_worker)
def request(self, method, host, path, headers, body, url="", timeout=60):
# xlog.debug("task start request")
if not url:
url = "%s %s%s" % (method, host, path)
self.last_request_time = time.time()
q = Queue.Queue()
task = http_common.Task(method, host, path, headers, body, q, url, timeout)
task.set_state("start_request")
self.request_queue.put(task)
# self.working_tasks[task.unique_id] = task
response = q.get(True)
task.set_state("get_response")
# del self.working_tasks[task.unique_id]
return response
def retry_task_cb(self, task, reason=""):
if task.responsed:
xlog.warn("retry but responsed. %s", task.url)
st = traceback.extract_stack()
stl = traceback.format_list(st)
xlog.warn("stack:%r", repr(stl))
task.put_data("")
return
if task.retry_count > 10:
task.response_fail("retry time exceed 10")
return
if time.time() - task.start_time > task.timeout:
task.response_fail("retry timeout:%d" % (time.time() - task.start_time))
return
task.set_state("retry(%s)" % reason)
task.retry_count += 1
self.request_queue.put(task)
def dispatcher(self):
while connect_control.keep_running:
start_time = time.time()
try:
task = self.request_queue.get(True)
if task is None:
# exit
break
except Exception as e:
xlog.exception("http_dispatcher dispatcher request_queue.get fail:%r", e)
continue
get_time = time.time()
get_cost = get_time - start_time
task.set_state("get_task(%d)" % get_cost)
try:
worker = self.get_worker()
except Exception as e:
xlog.warn("get worker fail:%r", e)
task.response_fail(reason="get worker fail:%r" % e)
continue
if worker is None:
# can send because exit.
xlog.warn("http_dispatcher get None worker")
task.response_fail("get worker fail.")
continue
get_worker_time = time.time()
get_cost = get_worker_time - get_time
task.set_state("get_worker(%d):%s" % (get_cost, worker.ip))
task.worker = worker
try:
worker.request(task)
except Exception as e:
xlog.exception("dispatch request:%r", e)
# wait up threads to exit.
self.wait_a_worker_cv.notify()
self.triger_create_worker_cv.notify()
def is_idle(self):
return time.time() - self.last_request_time > self.idle_time
def close_cb(self, worker):
try:
self.workers.remove(worker)
if worker.version == "2":
self.h2_num -= 1
else:
self.h1_num -= 1
except:
pass
def close_all_worker(self):
for w in self.workers:
w.close("close all worker")
self.workers = []
self.h1_num = 0
self.h2_num = 0
def to_string(self):
now = time.time()
worker_rate = {}
for w in self.workers:
worker_rate[w] = w.get_rtt_rate()
w_r = sorted(worker_rate.items(), key=operator.itemgetter(1))
out_str = 'thread num:%d\r\n' % threading.activeCount()
for w, r in w_r:
out_str += "%s rtt:%d a:%d live:%d inactive:%d processed:%d" % \
(w.ip, w.rtt, w.accept_task, (now-w.ssl_sock.create_time), (now-w.last_active_time), w.processed_tasks)
if w.version == "2":
out_str += " streams:%d ping_on_way:%d\r\n" % (len(w.streams), w.ping_on_way)
elif w.version == "1.1":
out_str += " Trace:%s" % w.get_trace()
out_str += "\r\n Speed:"
for speed in w.speed_history:
out_str += "%d," % speed
out_str += "\r\n"
out_str += "\r\n working_tasks:\r\n"
for unique_id in self.working_tasks:
task = self.working_tasks[unique_id]
out_str += task.to_string()
return out_str
class HttpsDispatcher(object):
idle_time = 2 * 60
def __init__(self,
logger,
config,
ip_manager,
connection_manager,
http1worker=Http1Worker,
http2worker=Http2Worker):
self.logger = logger
self.config = config
self.ip_manager = ip_manager
self.connection_manager = connection_manager
self.connection_manager.set_ssl_created_cb(self.on_ssl_created_cb)
self.http1worker = http1worker
self.http2worker = http2worker
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.last_request_time = time.time()
self.task_count_lock = threading.Lock()
self.task_count = 0
self.running = True
# for statistic
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.rtts = []
self.last_sent = self.total_sent = 0
self.last_received = self.total_received = 0
self.second_stats = Queue.deque()
self.last_statistic_time = time.time()
self.second_stat = {"rtt": 0, "sent": 0, "received": 0}
self.minute_stat = {"rtt": 0, "sent": 0, "received": 0}
self.trigger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = simple_queue.Queue()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
threading.Thread(target=self.connection_checker).start()
def stop(self):
self.running = False
self.request_queue.put(None)
self.close_all_worker("stop")
def on_ssl_created_cb(self, ssl_sock, check_free_work=True):
# self.logger.debug("on_ssl_created_cb %s", ssl_sock.ip)
if not self.running:
ssl_sock.close()
return
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
if ssl_sock.h2:
worker = self.http2worker(self.logger, self.ip_manager,
self.config, ssl_sock, self.close_cb,
self.retry_task_cb,
self._on_worker_idle_cb,
self.log_debug_data)
self.h2_num += 1
else:
worker = self.http1worker(self.logger, self.ip_manager,
self.config, ssl_sock, self.close_cb,
self.retry_task_cb,
self._on_worker_idle_cb,
self.log_debug_data)
self.h1_num += 1
self.workers.append(worker)
if check_free_work:
self.check_free_worker()
def _on_worker_idle_cb(self):
self.wait_a_worker_cv.notify()
def create_worker_thread(self):
while self.running:
self.trigger_create_worker_cv.wait()
try:
ssl_sock = self.connection_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# self.logger.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_work=False)
except Exception as e:
#self.logger.exception("on_ssl_created_cb except:%r", e)
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
def get_worker(self, nowait=False):
while self.running:
best_score = 99999999
best_worker = None
idle_num = 0
now = time.time()
for worker in self.workers:
if not worker.accept_task:
# self.logger.debug("not accept")
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
score = worker.get_score()
if best_score > score:
best_score = score
best_worker = worker
if len(self.workers) < self.config.dispather_max_workers and \
(best_worker is None or
idle_num < self.config.dispather_min_idle_workers or
len(self.workers) < self.config.dispather_min_workers or
(now - best_worker.last_recv_time) < self.config.dispather_work_min_idle_time or
best_score > self.config.dispather_work_max_score or
(best_worker.version == "2" and len(best_worker.streams) >= self.config.http2_target_concurrent)):
# self.logger.debug("trigger get more worker")
self.trigger_create_worker_cv.notify()
if nowait or \
(best_worker and (now - best_worker.last_recv_time) >= self.config.dispather_work_min_idle_time):
# self.logger.debug("return worker")
return best_worker
self.wait_a_worker_cv.wait(time.time() + 1)
# self.logger.debug("get wait_a_worker_cv")
#time.sleep(0.1)
def check_free_worker(self):
# close slowest worker,
# give change for better worker
while True:
slowest_score = 9999
slowest_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "2" and len(worker.streams) > 0:
continue
score = worker.get_score()
if score < 1000:
idle_num += 1
if score > slowest_score:
slowest_score = score
slowest_worker = worker
if idle_num < self.config.dispather_max_idle_workers or \
idle_num < int(len(self.workers) * 0.3) or \
len(self.workers) < self.config.dispather_max_workers:
return
if slowest_worker is None:
return
self.close_cb(slowest_worker)
def request(self, method, host, path, headers, body, url="", timeout=60):
if self.task_count > self.config.max_task_num:
self.logger.warn("task num exceed")
time.sleep(1)
return None
with self.task_count_lock:
self.task_count += 1
try:
# self.logger.debug("task start request")
if not url:
url = "%s %s%s" % (method, host, path)
self.last_request_time = time.time()
q = simple_queue.Queue()
task = http_common.Task(self.logger, self.config, method, host,
path, headers, body, q, url, timeout)
task.set_state("start_request")
self.request_queue.put(task)
response = q.get(timeout=timeout)
if response and response.status == 200:
self.success_num += 1
self.continue_fail_num = 0
else:
self.logger.warn("task %s %s %s timeout", method, host, path)
self.fail_num += 1
self.continue_fail_num += 1
self.last_fail_time = time.time()
task.set_state("get_response")
return response
finally:
with self.task_count_lock:
self.task_count -= 1
def retry_task_cb(self, task, reason=""):
self.fail_num += 1
self.continue_fail_num += 1
self.last_fail_time = time.time()
self.logger.warn("retry_task_cb: %s", task.url)
if task.responsed:
self.logger.warn("retry but responsed. %s", task.url)
st = traceback.extract_stack()
stl = traceback.format_list(st)
self.logger.warn("stack:%r", repr(stl))
task.finish()
return
if task.retry_count > 10:
task.response_fail("retry time exceed 10")
return
if time.time() - task.start_time > task.timeout:
task.response_fail("retry timeout:%d" %
(time.time() - task.start_time))
return
if not self.running:
task.response_fail("retry but stopped.")
return
task.set_state("retry(%s)" % reason)
task.retry_count += 1
self.request_queue.put(task)
def dispatcher(self):
while self.running:
start_time = time.time()
try:
task = self.request_queue.get(True)
if task is None:
# exit
break
except Exception as e:
self.logger.exception(
"http_dispatcher dispatcher request_queue.get fail:%r", e)
continue
get_time = time.time()
get_cost = get_time - start_time
task.set_state("get_task(%d)" % get_cost)
try:
worker = self.get_worker()
except Exception as e:
self.logger.warn("get worker fail:%r", e)
task.response_fail(reason="get worker fail:%r" % e)
continue
if worker is None:
# can send because exit.
self.logger.warn("http_dispatcher get None worker")
task.response_fail("get worker fail.")
continue
get_worker_time = time.time()
get_cost = get_worker_time - get_time
task.set_state("get_worker(%d):%s" % (get_cost, worker.ip_str))
task.worker = worker
try:
worker.request(task)
except Exception as e:
self.logger.exception("dispatch request:%r", e)
# wait up threads to exit.
self.wait_a_worker_cv.notify()
self.trigger_create_worker_cv.notify()
def connection_checker(self):
while self.running:
now = time.time()
try:
for worker in list(self.workers):
if worker.version == "1.1":
continue
worker.check_active(now)
except Exception as e:
self.logger.exception("check worker except:%r")
time.sleep(1)
def is_idle(self):
return time.time() - self.last_request_time > self.idle_time
def close_cb(self, worker):
try:
self.workers.remove(worker)
if worker.version == "2":
self.h2_num -= 1
else:
self.h1_num -= 1
except:
pass
def close_all_worker(self, reason="close all worker"):
for w in list(self.workers):
if w.accept_task:
w.close(reason)
self.workers = []
self.h1_num = 0
self.h2_num = 0
def log_debug_data(self, rtt, sent, received):
self.rtts.append(rtt)
self.total_sent += sent
self.total_received += received
def statistic(self):
now = time.time()
if now > self.last_statistic_time + 60:
rtt = 0
sent = 0
received = 0
for stat in self.second_stats:
rtt = max(rtt, stat["rtt"])
sent += stat["sent"]
received += stat["received"]
self.minute_stat = {"rtt": rtt, "sent": sent, "received": received}
self.second_stats = Queue.deque()
self.last_statistic_time = now
if len(self.rtts):
rtt = max(self.rtts)
else:
rtt = 0
self.second_stat = {
"rtt": rtt,
"sent": self.total_sent - self.last_sent,
"received": self.total_received - self.last_received
}
self.rtts = []
self.last_sent = self.total_sent
self.last_received = self.total_received
self.second_stats.append(self.second_stat)
def worker_num(self):
return len(self.workers)
def get_score(self):
now = time.time()
if now - self.last_fail_time < 60 and \
self.continue_fail_num > 10:
return None
worker = self.get_worker(nowait=True)
if not worker:
return None
return worker.get_score() * self.config.dispather_score_factor
def to_string(self):
now = time.time()
worker_rate = {}
for w in self.workers:
worker_rate[w] = w.get_rtt_rate()
w_r = sorted(worker_rate.items(), key=operator.itemgetter(1))
out_str = 'thread num:%d\r\n' % threading.activeCount()
for w, r in w_r:
out_str += "%s score:%d rtt:%d running:%d accept:%d live:%d inactive:%d processed:%d" % \
(w.ip_str, w.get_score(), w.rtt, w.keep_running, w.accept_task,
(now-w.ssl_sock.create_time), (now-w.last_recv_time), w.processed_tasks)
if w.version == "2":
out_str += " continue_timeout:%d streams:%d ping_on_way:%d remote_win:%d send_queue:%d\r\n" % \
(w.continue_timeout, len(w.streams), w.ping_on_way, w.remote_window_size, w.send_queue.qsize())
elif w.version == "1.1":
out_str += " Trace:%s" % w.get_trace()
out_str += "\r\n Speed:"
for speed in w.speed_history:
out_str += "%d," % speed
out_str += "\r\n"
out_str += "\r\n working_tasks:\r\n"
for unique_id in self.working_tasks:
task = self.working_tasks[unique_id]
out_str += task.to_string()
return out_str
class HttpsDispatcher(object):
idle_time = 20 * 60
def __init__(self):
self.request_queue = Queue.Queue()
self.workers = []
self.working_tasks = {}
self.h1_num = 0
self.h2_num = 0
self.create_worker_th = None
self.last_request_time = time.time()
self.triger_create_worker_cv = SimpleCondition()
self.wait_a_worker_cv = SimpleCondition()
threading.Thread(target=self.dispatcher).start()
threading.Thread(target=self.create_worker_thread).start()
# move created ssl to worker after ssl timeout
https_manager.set_ssl_time_handler(self.on_ssl_created_cb)
def on_ssl_created_cb(self, ssl_sock, check_free_worke=True):
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
appid = appid_manager.get_appid()
if not appid:
time.sleep(60)
ssl_sock.close()
raise http_common.GAE_Exception(1, "no appid can use")
ssl_sock.appid = appid
ssl_sock.host = ssl_sock.appid + ".appspot.com"
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb)
self.h1_num += 1
self.workers.append(worker)
self.wait_a_worker_cv.notify()
if check_free_worke:
self.check_free_worker()
def _on_worker_idle_cb(self):
self.wait_a_worker_cv.notify()
def create_worker_thread(self):
while connect_control.keep_running:
try:
ssl_sock = https_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# xlog.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_worke=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
if idle_num > 5 and acceptable_num > 20:
self.triger_create_worker_cv.wait()
def get_worker(self):
while connect_control.keep_running:
best_rtt = 9999
best_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
rtt = worker.get_rtt_rate()
if rtt < best_rtt:
best_rtt = rtt
best_worker = worker
if idle_num < 5:
self.triger_create_worker_cv.notify()
if best_worker:
return best_worker
self.wait_a_worker_cv.wait()
def check_free_worker(self):
# close slowest worker,
# give change for better worker
while True:
slowest_rtt = 9999
slowest_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "2" and len(worker.streams) > 0:
continue
idle_num += 1
rtt = worker.get_rtt_rate()
if rtt > slowest_rtt:
slowest_rtt = rtt
slowest_worker = worker
if idle_num < 30 or idle_num < int(len(self.workers) * 0.3):
return
if slowest_worker is None:
return
self.close_cb(slowest_worker)
def request(self, headers, body, url):
# xlog.debug("task start request")
self.last_request_time = time.time()
q = Queue.Queue()
task = http_common.Task(headers, body, q, url)
unique_id = task.unique_id
task.set_state("start_request")
self.request_queue.put(task)
self.working_tasks[task.unique_id] = task
response = q.get(True)
task.set_state("get_response")
try:
del self.working_tasks[task.unique_id]
except Exception as e:
xlog.error("http_dispatcher request unique_id %s, %s not found.", unique_id, task.unique_id)
return response
def retry_task_cb(self, task):
if task.responsed:
xlog.warn("retry but responsed. %s", task.url)
st = traceback.extract_stack()
stl = traceback.format_list(st)
xlog.warn("stack:%r", repr(stl))
task.put_data("")
return
if task.retry_count > 10:
task.response_fail("retry time exceed 10")
return
if time.time() - task.start_time > 240:
task.response_fail("retry timeout:%d" % (time.time() - task.start_time))
return
task.set_state("retry")
task.retry_count += 1
self.request_queue.put(task)
def dispatcher(self):
while connect_control.keep_running:
try:
task = self.request_queue.get(True)
if task is None:
# exit
break
except Exception as e:
xlog.exception("http_dispatcher dispatcher request_queue.get fail:%r", e)
continue
task.set_state("get_task")
try:
worker = self.get_worker()
except Exception as e:
xlog.warn("get worker fail:%r", e)
task.response_fail(reason="get worker fail:%r" % e)
continue
if worker is None:
# can send because exit.
xlog.warn("http_dispatcher get None worker")
continue
task.set_state("get_worker:%s" % worker.ip)
try:
worker.request(task)
except Exception as e:
xlog.exception("dispatch request:%r", e)
# wait up threads to exit.
self.wait_a_worker_cv.notify()
self.triger_create_worker_cv.notify()
def is_idle(self):
return time.time() - self.last_request_time > self.idle_time
def close_cb(self, worker):
try:
self.workers.remove(worker)
if worker.version == "2":
self.h2_num -= 1
else:
self.h1_num -= 1
except:
pass
def close_all_worker(self):
for w in self.workers:
w.close("close all worker")
self.workers = []
self.h1_num = 0
self.h2_num = 0
def to_string(self):
worker_rate = {}
for w in self.workers:
worker_rate[w] = w.get_rtt_rate()
w_r = sorted(worker_rate.items(), key=operator.itemgetter(1))
out_str = 'thread num:%d\r\n' % threading.activeCount()
for w, r in w_r:
out_str += "%s rtt:%d a:%d live:%d processed:%d" % \
(w.ip, w.rtt, w.accept_task, (time.time()-w.ssl_sock.create_time), w.processed_tasks)
if w.version == "2":
out_str += " streams:%d ping_on_way:%d\r\n" % (len(w.streams), w.ping_on_way)
out_str += " Speed:"
for speed in w.speed_history:
out_str += "%d," % speed
out_str += "\r\n"
out_str += "\r\n<br> working_tasks:\r\n"
for unique_id in self.working_tasks:
task = self.working_tasks[unique_id]
out_str += task.to_string()
return out_str
