def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
relay_id = str(uuid4())
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
connect_path = '/' + '/'.join(path_str_ls)
if parsed_req_url.path.endswith('/'):
connect_path = connect_path + '/'
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse(
(parsed_req_url.scheme, relay_addr + ':' + relay_port,
connect_path, parsed_req_url.params, parsed_req_url.query,
parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
if req.headers.has_key('x-object-manifest'):
object_manifest = req.headers['x-object-manifest']
cont = object_manifest.split('/')[0]
obj = object_manifest.split('/')[1:]
cont_parts = cont.split(':')
if len(cont_parts) >= 2:
real_cont = ':'.join(cont_parts[1:])
object_manifest = real_cont + '/' + '/'.join(obj)
req.headers['x-object-manifest'] = object_manifest
original_url = req.url
self.logger.info(
'Request[%s]: %s %s with headers = %s, Connect to %s (via %s)'
% (str(relay_id), req.method, req.url, req.headers,
connect_url, proxy))
result = RelayRequest(self.conf,
req,
connect_url,
proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info(
'Retry Req[%s]: %s %s with headers = %s, Connect to %s (via %s)'
% (str(relay_id), req.method, req.url, req.headers,
connect_url, proxy))
continue
else:
return result
if result.getheader('location'):
location = result.getheader('location')
parsed_location = urlparse(location)
parsed_connect_url = urlparse(connect_url)
if parsed_location.netloc.startswith(
parsed_connect_url.netloc):
parsed_orig_url = urlparse(original_url)
loc_prefix = parsed_orig_url.path.split('/')[1]
if parsed_orig_url.path.split(
'/')[1] != self.req_version_str:
rewrited_path = '/' + loc_prefix + parsed_location.path
else:
rewrited_path = parsed_location.path
rewrited_location = (parsed_orig_url.scheme,
parsed_orig_url.netloc, rewrited_path,
parsed_location.params,
parsed_location.query,
parsed_location.fragment)
response = Response(status='%s %s' %
(result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(
response,
{'Content-Length': result.getheader('Content-Length')})
if result.getheader('location'):
update_headers(response,
{'Location': urlunparse(rewrited_location)})
response.status = result.status
self.logger.info('Response[%s]: %s by %s %s %s' %
(str(relay_id), response.status, req.method,
req.url, response.headers))
return response
def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
relay_id = str(uuid4())
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
connect_path = '/' + '/'.join(path_str_ls)
if parsed_req_url.path.endswith('/'):
connect_path = connect_path + '/'
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse((parsed_req_url.scheme,
relay_addr + ':' + relay_port,
connect_path,
parsed_req_url.params,
parsed_req_url.query,
parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
if req.headers.has_key('x-object-manifest'):
object_manifest = req.headers['x-object-manifest']
cont = object_manifest.split('/')[0]
obj = object_manifest.split('/')[1:]
cont_parts = cont.split(':')
if len(cont_parts) >= 2:
real_cont = ':'.join(cont_parts[1:])
object_manifest = real_cont + '/' + '/'.join(obj)
req.headers['x-object-manifest'] = object_manifest
original_url = req.url
self.logger.info('Request[%s]: %s %s with headers = %s, Connect to %s (via %s)' %
(str(relay_id),
req.method, req.url, req.headers,
connect_url, proxy))
result = RelayRequest(self.conf, req, connect_url, proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info('Retry Req[%s]: %s %s with headers = %s, Connect to %s (via %s)' %
(str(relay_id),
req.method, req.url, req.headers,
connect_url, proxy))
continue
else:
return result
if result.getheader('location'):
location = result.getheader('location')
parsed_location = urlparse(location)
parsed_connect_url = urlparse(connect_url)
if parsed_location.netloc.startswith(parsed_connect_url.netloc):
parsed_orig_url = urlparse(original_url)
loc_prefix = parsed_orig_url.path.split('/')[1]
if parsed_orig_url.path.split('/')[1] != self.req_version_str:
rewrited_path = '/' + loc_prefix + parsed_location.path
else:
rewrited_path = parsed_location.path
rewrited_location = (parsed_orig_url.scheme,
parsed_orig_url.netloc,
rewrited_path,
parsed_location.params,
parsed_location.query,
parsed_location.fragment)
response = Response(status='%s %s' % (result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(response, {'Content-Length':
result.getheader('Content-Length')})
if result.getheader('location'):
update_headers(response, {'Location': urlunparse(rewrited_location)})
response.status = result.status
self.logger.info('Response[%s]: %s by %s %s %s' %
(str(relay_id),
response.status, req.method, req.url,
response.headers))
return response
def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse((parsed_req_url.scheme,
relay_addr + ':' + relay_port,
'/' + '/'.join(path_str_ls),
parsed_req_url.params,
parsed_req_url.query,
parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
self.logger.debug('Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
result = RelayRequest(self.conf, req, connect_url, proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info('Retry Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
continue
else:
return result
response = Response(status='%s %s' % (result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(response, {'Content-Length':
result.getheader('Content-Length')})
response.status = result.status
return response
def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse(
(parsed_req_url.scheme, relay_addr + ':' + relay_port,
'/' + '/'.join(path_str_ls), parsed_req_url.params,
parsed_req_url.query, parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
self.logger.debug('Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
result = RelayRequest(self.conf,
req,
connect_url,
proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info('Retry Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
continue
else:
return result
response = Response(status='%s %s' %
(result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(
response,
{'Content-Length': result.getheader('Content-Length')})
response.status = result.status
return response
