def load_from_cdx_test(cdx, revisit_func=load_orig_cdx, reraise=False,
failed_files=None):
resolve_loader = ResolvingLoader(test_warc_dir)
cdx = CDXObject(cdx)
try:
(headers, stream) = resolve_loader.resolve_headers_and_payload(cdx,
failed_files,
revisit_func)
print headers
sys.stdout.write(stream.readline())
sys.stdout.write(stream.readline())
except ArchiveLoadFailed as e:
if reraise:
raise
else:
print 'Exception: ' + e.__class__.__name__
def _init_replay_view(self, config):
cookie_maker = config.get('cookie_maker')
record_loader = ArcWarcRecordLoader(cookie_maker=cookie_maker)
paths = config.get('archive_paths')
resolving_loader = ResolvingLoader(PathResolverMapper()(paths),
record_loader=record_loader)
return ReplayView(resolving_loader, config)
def _init_replay_view(self, config):
cookie_maker = config.get('cookie_maker')
record_loader = ArcWarcRecordLoader(cookie_maker=cookie_maker)
paths = config.get('archive_paths')
resolving_loader = ResolvingLoader(PathResolverMapper()(paths),
record_loader=record_loader)
redis_warc_resolver = config.get('redis_warc_resolver')
if redis_warc_resolver:
resolving_loader.path_resolvers.append(redis_warc_resolver)
return WebRecReplayView(resolving_loader, config)
def __init__(self, query_handler, config=None):
super(WBHandler, self).__init__(config)
self.index_reader = query_handler
cookie_maker = config.get('cookie_maker')
record_loader = ArcWarcRecordLoader(cookie_maker=cookie_maker)
paths = config.get('archive_paths')
resolving_loader = ResolvingLoader(paths=paths,
record_loader=record_loader)
self.replay = ReplayView(resolving_loader, config)
self.fallback_handler = None
self.fallback_name = config.get('fallback')
def load_from_cdx_test(cdx,
revisit_func=load_orig_cdx,
reraise=False,
failed_files=None):
resolve_loader = ResolvingLoader(PathResolverMapper()(test_warc_dir))
cdx = CDXObject(cdx.encode('utf-8'))
try:
(headers, stream) = resolve_loader(cdx, failed_files, revisit_func)
print(repr(headers))
sys.stdout.write(stream.readline().decode('utf-8'))
sys.stdout.write(stream.readline().decode('utf-8'))
except ArchiveLoadFailed as e:
if reraise:
raise
else:
print('Exception: ' + e.__class__.__name__)