class WARCPathLoader(DefaultResolverMixin, BaseLoader): def __init__(self, paths, cdx_source): self.paths = paths self.resolvers = self.make_resolvers(self.paths) self.resolve_loader = ResolvingLoader(self.resolvers, no_record_parse=True) self.headers_parser = StatusAndHeadersParser([], verify=False) self.cdx_source = cdx_source def load_resource(self, cdx, params): if cdx.get('_cached_result'): return cdx.get('_cached_result') if not cdx.get('filename') or cdx.get('offset') is None: return None orig_source = cdx.get('source', '').split(':')[0] formatter = ParamFormatter(params, orig_source) cdx._formatter = formatter def local_index_query(local_params): for n, v in six.iteritems(params): if n.startswith('param.'): local_params[n] = v cdx_iter, errs = self.cdx_source(local_params) for cdx in cdx_iter: cdx._formatter = formatter yield cdx failed_files = [] headers, payload = (self.resolve_loader.load_headers_and_payload( cdx, failed_files, local_index_query)) http_headers_buff = None if payload.rec_type in ('response', 'revisit'): status = cdx.get('status') # status may not be set for 'revisit' if not status or status.startswith('3'): http_headers = self.headers_parser.parse(payload.raw_stream) try: self.raise_on_self_redirect( params, cdx, http_headers.get_statuscode(), http_headers.get_header('Location')) except LiveResourceException: headers.raw_stream.close() payload.raw_stream.close() raise http_headers_buff = http_headers.to_bytes() warc_headers = payload.rec_headers if headers != payload: warc_headers.replace_header( 'WARC-Refers-To-Target-URI', payload.rec_headers.get_header('WARC-Target-URI')) warc_headers.replace_header( 'WARC-Refers-To-Date', payload.rec_headers.get_header('WARC-Date')) warc_headers.replace_header( 'WARC-Target-URI', headers.rec_headers.get_header('WARC-Target-URI')) warc_headers.replace_header( 'WARC-Date', headers.rec_headers.get_header('WARC-Date')) headers.raw_stream.close() return (warc_headers, http_headers_buff, payload.raw_stream) def __str__(self): return 'WARCPathLoader'
class WARCPathLoader(DefaultResolverMixin, BaseLoader): def __init__(self, paths, cdx_source): self.paths = paths self.resolvers = self.make_resolvers(self.paths) self.resolve_loader = ResolvingLoader(self.resolvers, no_record_parse=True) self.headers_parser = StatusAndHeadersParser([], verify=False) self.cdx_source = cdx_source def load_resource(self, cdx, params): if cdx.get('_cached_result'): return cdx.get('_cached_result') if not cdx.get('filename') or cdx.get('offset') is None: return None orig_source = cdx.get('source', '').split(':')[0] formatter = ParamFormatter(params, orig_source) cdx._formatter = formatter def local_index_query(local_params): for n, v in six.iteritems(params): if n.startswith('param.'): local_params[n] = v cdx_iter, errs = self.cdx_source(local_params) for cdx in cdx_iter: cdx._formatter = formatter yield cdx failed_files = [] headers, payload = (self.resolve_loader.load_headers_and_payload( cdx, failed_files, local_index_query)) http_headers_buff = None if payload.rec_type in ('response', 'revisit'): status = cdx.get('status') # if status is not set and not, 2xx, 4xx, 5xx # go through self-redirect check just in case if not status or not status.startswith(('2', '4', '5')): http_headers = self.headers_parser.parse(payload.raw_stream) try: orig_size = payload.raw_stream.tell() except: orig_size = 0 try: self.raise_on_self_redirect( params, cdx, http_headers.get_statuscode(), http_headers.get_header('Location')) except LiveResourceException: no_except_close(headers.raw_stream) no_except_close(payload.raw_stream) raise http_headers_buff = http_headers.to_bytes() # if new http_headers_buff is different length, # attempt to adjust content-length on the WARC record if orig_size and len(http_headers_buff) != orig_size: orig_cl = payload.rec_headers.get_header('Content-Length') if orig_cl: new_cl = int(orig_cl) + (len(http_headers_buff) - orig_size) payload.rec_headers.replace_header( 'Content-Length', str(new_cl)) warc_headers = payload.rec_headers if headers != payload: warc_headers.replace_header( 'WARC-Refers-To-Target-URI', payload.rec_headers.get_header('WARC-Target-URI')) warc_headers.replace_header( 'WARC-Refers-To-Date', payload.rec_headers.get_header('WARC-Date')) warc_headers.replace_header( 'WARC-Target-URI', headers.rec_headers.get_header('WARC-Target-URI')) warc_headers.replace_header( 'WARC-Date', headers.rec_headers.get_header('WARC-Date')) no_except_close(headers.raw_stream) return (warc_headers, http_headers_buff, payload.raw_stream) def __str__(self): return 'WARCPathLoader'