def create_worker_thread(self):
while connect_control.keep_running:
try:
ssl_sock = https_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# xlog.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_worke=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
if idle_num > 5 and acceptable_num > 20:
self.triger_create_worker_cv.wait()
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.warn('request failed:%s', e)
if ssl_sock:
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def create_worker_thread(self):
while connect_control.keep_running:
try:
ssl_sock = https_manager.get_ssl_connection()
except Exception as e:
continue
if not ssl_sock:
# xlog.warn("create_worker_thread get ssl_sock fail")
continue
try:
self.on_ssl_created_cb(ssl_sock, check_free_worke=False)
except:
time.sleep(10)
idle_num = 0
acceptable_num = 0
for worker in self.workers:
if worker.accept_task:
acceptable_num += 1
if worker.version == "1.1":
if worker.accept_task:
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
if idle_num > 5 and acceptable_num > 20:
self.triger_create_worker_cv.wait()
def fetch(method, host, path, headers, payload, bufsize=8192):
request_data = '%s %s HTTP/1.1\r\n' % (method, path)
request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items())
request_data += '\r\n'
ssl_sock = https_manager.get_ssl_connection(host)
if not ssl_sock:
return
ssl_sock.send(request_data.encode())
payload_len = len(payload)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = ssl_sock.send(payload[start:start+send_size])
start += sended
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.ssl_sock = ssl_sock
try:
orig_timeout = ssl_sock.gettimeout()
ssl_sock.settimeout(90)
response.begin()
ssl_sock.settimeout(orig_timeout)
except httplib.BadStatusLine as e:
xlog.warn("direct_handler.fetch bad status line:%r", e)
response = None
except Exception as e:
xlog.warn("direct_handler.fetch:%r", e)
return response
def get_worker(self):
best_rtt = 9999
best_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
rtt = worker.get_rtt_rate()
if rtt < best_rtt:
best_rtt = rtt
best_worker = worker
if idle_num == 0 or len(self.workers) < self.min_worker_num:
self.create_more_worker()
if best_worker:
return best_worker
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
raise GAE_Exception(1, "no ssl_sock")
worker = self.on_ssl_created_cb(ssl_sock)
return worker
def fetch(method, host, path, headers, payload, bufsize=8192):
request_data = '%s %s HTTP/1.1\r\n' % (method, path)
request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items())
request_data += '\r\n'
ssl_sock = https_manager.get_ssl_connection(host)
if not ssl_sock:
return
try:
ssl_sock.send(request_data.encode())
payload_len = len(payload)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = ssl_sock.send(payload[start:start+send_size])
start += sended
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.ssl_sock = ssl_sock
orig_timeout = ssl_sock.gettimeout()
ssl_sock.settimeout(90)
response.begin()
ssl_sock.settimeout(orig_timeout)
except httplib.BadStatusLine as e:
xlog.warn("direct_handler.fetch bad status line:%r", e)
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
response = None
except Exception as e:
xlog.warn("direct_handler.fetch:%r", e)
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
response = None
return response
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug("create_ssl_connection fail")
continue
if ssl_sock.host == "":
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
raise GAE_Exception(1, "no appid can use")
headers["Host"] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers["Host"]
else:
headers["Host"] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.warn("request failed:%s", e)
if ssl_sock:
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def get_worker(self):
best_rtt = 9999
best_worker = None
idle_num = 0
for worker in self.workers:
if not worker.accept_task:
continue
if worker.version == "1.1":
idle_num += 1
else:
if len(worker.streams) == 0:
idle_num += 1
rtt = worker.get_rtt_rate()
if rtt < best_rtt:
best_rtt = rtt
best_worker = worker
if idle_num == 0:
self.create_more_worker()
if best_worker:
return best_worker
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
raise GAE_Exception(1, "no ssl_sock")
worker = self.on_ssl_created_cb(ssl_sock)
return worker
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
google_ip.report_connect_closed(ssl_sock.ip, "no appid")
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.exception('request failed:%s', e)
if ssl_sock:
google_ip.report_connect_closed(ssl_sock.ip, "request_except")
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def create_worker_thread(self):
try:
while True:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
#xlog.warn("create_worker_thread get ssl_sock fail")
continue
self.on_ssl_created_cb(ssl_sock)
if len(self.workers) > self.min_worker_num:
break
finally:
self.create_worker_th = None
def create_worker_thread(self):
try:
while True:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
#xlog.warn("create_worker_thread get ssl_sock fail")
continue
self.on_ssl_created_cb(ssl_sock)
if len(self.workers) > self.min_worker_num:
break
finally:
self.create_worker_th = None
def create_worker_thread(self):
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.warn("create_worker_thread get ssl_sock fail")
self.on_ssl_created_cb(ssl_sock)
def create_worker_thread(self):
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.warn("create_worker_thread get ssl_sock fail")
self.on_ssl_created_cb(ssl_sock)