def request_gae_server(headers, body):
# process on http protocol
# process status code return by http server
# raise error, let up layer retry.
response = http_dispatch.request(headers, body)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(response.status,
"fetch gae fail:%d" % response.status)
server_type = response.headers.get("server", "")
content_type = response.headers.get("content-type", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
# some ip can connect, and server type can be gws
# but can't use as GAE server
# so we need remove it immediately
xlog.warn("IP:%s not support GAE, server:%s status:%d",
response.ssl_sock.ip, server_type, response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid,
response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s', appid,
response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip,
response.ssl_sock.appid, response.status)
else:
xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num,
google_ip.bad_ip_num)
return response
def request_gae_server(headers, body, url, timeout):
# process on http protocol
# process status code return by http server
# raise error, let up layer retry.
response = http_dispatch.request(headers, body, url, timeout)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(
response.status, "fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
# content_type = response.getheaders("content-type", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
# some ip can connect, and server type can be gws
# but can't use as GAE server
# so we need remove it immediately
xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers,
response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(
response.ssl_sock.appid, response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s',
appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip,
response.ssl_sock.appid, response.status)
return response
def request_gae_server(headers, body, url, timeout):
response = http_dispatch.request(headers, body, url, timeout)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(response.status,
"fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
xlog.warn("IP:%s not support GAE, headers:%s status:%d",
response.ssl_sock.ip, response.headers, response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid,
response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s', appid,
response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip,
response.ssl_sock.appid, response.status)
else:
xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num,
google_ip.bad_ip_num)
return response
def request_gae_server(headers, body, url, timeout):
response = http_dispatch.request(headers, body, url, timeout)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(
response.status, "fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers,
response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(
response.ssl_sock.appid, response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s',
appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status)
else:
xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num)
return response
def fetch_by_gae(method, url, headers, body):
if isinstance(body, basestring) and body:
if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
zbody = deflate(body)
if len(zbody) < len(body):
body = zbody
headers['Content-Encoding'] = 'deflate'
if len(body) > 10 * 1024 * 1024:
xlog.warn("body len:%d %s %s", len(body), method, url)
headers['Content-Length'] = str(len(body))
# GAE don't allow set `Host` header
if 'Host' in headers:
del headers['Host']
kwargs = {}
if config.GAE_PASSWORD:
kwargs['password'] = config.GAE_PASSWORD
#kwargs['options'] =
#kwargs['validate'] =
kwargs['maxsize'] = config.AUTORANGE_MAXSIZE
kwargs['timeout'] = '19'
payload = '%s %s HTTP/1.1\r\n' % (method, url)
payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items()
if k not in skip_headers)
#for k, v in headers.items():
# xlog.debug("Send %s: %s", k, v)
payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v)
for k, v in kwargs.items() if v)
request_headers = {}
payload = deflate(payload)
body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body)
request_headers['Content-Length'] = str(len(body))
response = http_dispatch.request(request_headers, body)
response.app_status = response.status
response.app_headers = response.headers
if response.app_status != 200:
return response
check_local_network.report_network_ok()
try:
data = response.body.get(2)
if len(data) < 2:
xlog.warn("fetch too short lead byte len:%d %s", len(data), url)
response.app_status = 502
# 502: Bad gateway
response.fp = io.BytesIO(
b'connection aborted. too short lead byte data=' + data)
response.read = response.fp.read
return response
headers_length, = struct.unpack('!h', data)
data = response.body.get(headers_length)
if len(data) < headers_length:
xlog.warn("fetch too short header need:%d get:%d %s",
headers_length, len(data), url)
response.app_status = 509
response.fp = io.BytesIO(
b'connection aborted. too short headers data=' + data)
response.read = response.fp.read
return response
raw_response_line, headers_data = inflate(data).split('\r\n', 1)
_, response.status, response.reason = raw_response_line.split(None, 2)
response.status = int(response.status)
response.reason = response.reason.strip()
headers_pairs = headers_data.split('\r\n')
response.headers = {}
for pair in headers_pairs:
if not pair:
break
k, v = pair.split(': ', 1)
response.headers[k] = v
return response
except Exception as e:
raise GAE_Exception("unpack protocol:%r", e)
def fetch_by_gae(method, url, headers, body):
if isinstance(body, basestring) and body:
if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
zbody = deflate(body)
if len(zbody) < len(body):
body = zbody
headers['Content-Encoding'] = 'deflate'
if len(body) > 10 * 1024 * 1024:
xlog.warn("body len:%d %s %s", len(body), method, url)
headers['Content-Length'] = str(len(body))
# GAE don't allow set `Host` header
if 'Host' in headers:
del headers['Host']
kwargs = {}
if config.GAE_PASSWORD:
kwargs['password'] = config.GAE_PASSWORD
#kwargs['options'] =
#kwargs['validate'] =
kwargs['maxsize'] = config.AUTORANGE_MAXSIZE
kwargs['timeout'] = '19'
payload = '%s %s HTTP/1.1\r\n' % (method, url)
payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in skip_headers)
#for k, v in headers.items():
# xlog.debug("Send %s: %s", k, v)
payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v)
request_headers = {}
payload = deflate(payload)
body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body)
request_headers['Content-Length'] = str(len(body))
response = http_dispatch.request(request_headers, body)
response.app_status = response.status
response.app_headers = response.headers
if response.app_status != 200:
return response
check_local_network.report_network_ok()
try:
data = response.body.get(2)
if len(data) < 2:
xlog.warn("fetch too short lead byte len:%d %s", len(data), url)
response.app_status = 502
# 502: Bad gateway
response.fp = io.BytesIO(b'connection aborted. too short lead byte data=' + data)
response.read = response.fp.read
return response
headers_length, = struct.unpack('!h', data)
data = response.body.get(headers_length)
if len(data) < headers_length:
xlog.warn("fetch too short header need:%d get:%d %s", headers_length, len(data), url)
response.app_status = 509
response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data)
response.read = response.fp.read
return response
raw_response_line, headers_data = inflate(data).split('\r\n', 1)
_, response.status, response.reason = raw_response_line.split(None, 2)
response.status = int(response.status)
response.reason = response.reason.strip()
headers_pairs = headers_data.split('\r\n')
response.headers = {}
for pair in headers_pairs:
if not pair:
break
k, v = pair.split(': ', 1)
response.headers[k] = v
return response
except Exception as e:
raise GAE_Exception("unpack protocol:%r", e)