def _get_request(self, chunk):
"""
Connect to a chunk, fetch headers but don't read data.
Save the response object in `self.sources` list.
"""
try:
with green.ConnectionTimeout(self.connection_timeout):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
conn = http_connect(parsed.netloc, 'GET', parsed.path,
self.request_headers)
with green.OioTimeout(self.read_timeout):
source = conn.getresponse()
source.conn = conn
except (Exception, Timeout) as error:
logger.exception('Connection failed to %s', chunk)
self._resp_by_chunk[chunk["url"]] = (0, str(error))
return False
if source.status in (200, 206):
self.status = source.status
self._headers = source.getheaders()
self.sources.append((source, chunk))
return True
else:
logger.warn("Invalid response from %s: %d %s", chunk,
source.status, source.reason)
self._resp_by_chunk[chunk["url"]] = (source.status,
str(source.reason))
return False
def _get_request(self, chunk):
"""
Connect to a chunk, fetch headers but don't read data.
Save the response object in `self.sources` list.
"""
try:
with green.ConnectionTimeout(self.connection_timeout):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
if self.perfdata is not None:
connect_start = monotonic_time()
conn = http_connect(parsed.netloc,
'GET',
parsed.path,
self.request_headers,
scheme=parsed.scheme)
if self.perfdata is not None:
connect_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault('rawx', dict())
rawx_perfdata['connect.' + chunk['url']] = \
connect_end - connect_start
with green.OioTimeout(self.read_timeout):
source = conn.getresponse()
source.conn = conn
if self.perfdata is not None:
source.download_start = monotonic_time()
except (SocketError, Timeout) as err:
self.logger.error('Connection failed to %s (reqid=%s): %s', chunk,
self.reqid, err)
self._resp_by_chunk[chunk["url"]] = (0, text_type(err))
return False
except Exception as err:
self.logger.exception('Connection failed to %s (reqid=%s)', chunk,
self.reqid)
self._resp_by_chunk[chunk["url"]] = (0, text_type(err))
return False
if source.status in (200, 206):
self.status = source.status
self._headers = [(k.lower(), v) for k, v in source.getheaders()]
self.sources.append((source, chunk))
return True
else:
self.logger.warn("Invalid response from %s (reqid=%s): %d %s",
chunk, self.reqid, source.status, source.reason)
self._resp_by_chunk[chunk["url"]] = (source.status,
text_type(source.reason))
close_source(source, self.logger)
return False