def _init_warc_headers(self, uri, record_type, warc_headers_dict):
warc_headers = StatusAndHeaders(self.warc_version, list(warc_headers_dict.items()))
warc_headers.replace_header('WARC-Type', record_type)
if not warc_headers.get_header('WARC-Record-ID'):
warc_headers.add_header('WARC-Record-ID', self._make_warc_id())
if uri:
warc_headers.replace_header('WARC-Target-URI', uri)
if not warc_headers.get_header('WARC-Date'):
warc_headers.add_header('WARC-Date', self._make_warc_date())
return warc_headers
def parse_response(self, url, response, ip=None):
headers = []
payload = BytesIO()
content = response['content'].get('text', '')
if not content and not response.get('headers'):
self.logger.info('No headers or payload for: {0}'.format(url))
headers.append(('Content-Length', '0'))
if response['content'].get('encoding') == 'base64':
payload.write(base64.b64decode(content))
else:
payload.write(content.encode('utf-8'))
length = payload.tell()
payload.seek(0)
SKIP_HEADERS = ('content-encoding', 'transfer-encoding')
http2 = False
for header in response['headers']:
if header['name'].lower() not in SKIP_HEADERS:
headers.append((header['name'], header['value']))
#TODO: http2 detection -- write as same warc header?
if (not http2 and
header['name'] in (':method', ':scheme', ':path')):
http2 = True
status = response.get('status') or 204
reason = response.get('statusText')
if not reason:
reason = http_status_names.get(status, 'No Reason')
status_line = str(status) + ' ' + reason
proto = self._get_http_version(response)
http_headers = StatusAndHeaders(status_line, headers, protocol=proto)
if not content:
content_length = http_headers.get_header('Content-Length', '0')
if content_length != '0':
self.logger.info('No Content for length {0} {1}'.format(content_length, url))
http_headers.replace_header('Content-Length', '0')
else:
http_headers.replace_header('Content-Length', str(length))
warc_headers_dict = {}
if ip:
warc_headers_dict['WARC-IP-Address'] = ip
record = self.writer.create_warc_record(url, 'response',
http_headers=http_headers,
payload=payload,
length=length,
warc_headers_dict=warc_headers_dict)
return record
