def _http_request(self, chunkurl, method, body, headers, trailers=None):
parsed = urlparse(chunkurl)
if method == 'PUT':
headers['transfer-encoding'] = 'chunked'
if trailers:
headers['Trailer'] = list()
for k in trailers.values():
headers['Trailer'].append(k)
conn = http_connect(parsed.netloc, method, parsed.path,
headers)
if method == 'PUT':
if body:
conn.send(b'%x\r\n%s\r\n' % (len(body), body))
conn.send(b'0\r\n')
if trailers:
for k, v in trailers.items():
conn.send(b'%s: %s\r\n' % (k.encode('utf-8'),
v.encode('utf-8')))
conn.send(b'\r\n')
if method == 'PUT':
del headers['transfer-encoding']
if trailers:
del headers['Trailer']
resp = conn.getresponse()
body = resp.read()
conn.close()
return resp, body
def _get_request(self, chunk):
"""
Connect to a chunk, fetch headers but don't read data.
Save the response object in `self.sources` list.
"""
try:
with green.ConnectionTimeout(self.connection_timeout):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
conn = http_connect(parsed.netloc, 'GET', parsed.path,
self.request_headers)
with green.OioTimeout(self.read_timeout):
source = conn.getresponse()
source.conn = conn
except (Exception, Timeout) as error:
logger.exception('Connection failed to %s', chunk)
self._resp_by_chunk[chunk["url"]] = (0, str(error))
return False
if source.status in (200, 206):
self.status = source.status
self._headers = source.getheaders()
self.sources.append((source, chunk))
return True
else:
logger.warn("Invalid response from %s: %d %s", chunk,
source.status, source.reason)
self._resp_by_chunk[chunk["url"]] = (source.status,
str(source.reason))
return False
def _get_request(self, chunk):
"""
Connect to a chunk, fetch headers but don't read data.
Save the response object in `self.sources` list.
"""
try:
with green.ConnectionTimeout(self.connection_timeout):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
if self.perfdata is not None:
connect_start = monotonic_time()
conn = http_connect(parsed.netloc,
'GET',
parsed.path,
self.request_headers,
scheme=parsed.scheme)
if self.perfdata is not None:
connect_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['connect.' + chunk['url']] = \
connect_end - connect_start
with green.OioTimeout(self.read_timeout):
source = conn.getresponse()
source.conn = conn
if self.perfdata is not None:
source.download_start = monotonic_time()
except (SocketError, Timeout) as err:
self.logger.error('Connection failed to %s (reqid=%s): %s', chunk,
self.reqid, err)
self._resp_by_chunk[chunk["url"]] = (0, text_type(err))
return False
except Exception as err:
self.logger.exception('Connection failed to %s (reqid=%s)', chunk,
self.reqid)
self._resp_by_chunk[chunk["url"]] = (0, text_type(err))
return False
if source.status in (200, 206):
self.status = source.status
self._headers = [(k.lower(), v) for k, v in source.getheaders()]
self.sources.append((source, chunk))
return True
else:
self.logger.warn("Invalid response from %s (reqid=%s): %d %s",
chunk, self.reqid, source.status, source.reason)
self._resp_by_chunk[chunk["url"]] = (source.status,
text_type(source.reason))
close_source(source, self.logger)
return False
