def make_live_app():
app = BaseWarcServer()
app.add_route(
'/live',
DefaultResourceHandler(
SimpleAggregator({'live': LiveIndexSource()})))
return app
def load_auto_colls(self):
if not self.root_dir:
print('No Root Dir, Skip Auto Colls!')
return
dir_source = CacheDirectoryIndexSource(base_prefix=self.root_dir,
base_dir=self.index_paths,
config=self.config)
access_checker = AccessChecker(
CacheDirectoryAccessSource(self.acl_paths), self.default_access)
if self.dedup_index_url:
source = SimpleAggregator({
'dedup':
RedisMultiKeyIndexSource(self.dedup_index_url),
'dir':
dir_source
})
else:
source = dir_source
return DefaultResourceHandler(source,
self.archive_paths,
rules_file=self.rules_file,
access_checker=access_checker)
def load_coll(self, name, coll_config):
if coll_config == '$all' and self.auto_handler:
return self.auto_handler
if isinstance(coll_config, str):
index = coll_config
archive_paths = None
acl_paths = None
default_access = self.default_access
elif isinstance(coll_config, dict):
index = coll_config.get('index')
if not index:
index = coll_config.get('index_paths')
archive_paths = coll_config.get('archive_paths')
acl_paths = coll_config.get('acl_paths')
default_access = coll_config.get('default_access',
self.default_access)
surt_ordered = coll_config.get('surt_ordered', True)
else:
raise Exception('collection config must be string or dict')
# INDEX CONFIG
if index:
agg = init_index_agg({name: index}, surt_ordered=surt_ordered)
else:
if not isinstance(coll_config, dict):
raise Exception('collection config missing')
sequence = coll_config.get('sequence')
if sequence:
return self.init_sequence(name, sequence)
index_group = coll_config.get('index_group')
if not index_group:
raise Exception('no index, index_group or sequence found')
timeout = int(coll_config.get('timeout', 0))
agg = init_index_agg(index_group,
True,
timeout,
surt_ordered=surt_ordered)
# ARCHIVE CONFIG
if not archive_paths:
archive_paths = self.config.get('archive_paths')
# ACCESS CONFIG
access_checker = None
if acl_paths:
access_checker = AccessChecker(acl_paths, default_access)
return DefaultResourceHandler(agg,
archive_paths,
rules_file=self.rules_file,
access_checker=access_checker)
def setup(self):
app = BaseWarcServer()
base_url = 'http://localhost:{0}'.format(self.server.port)
app.add_route(
'/upstream',
DefaultResourceHandler(
SimpleAggregator(
{'upstream': UpstreamAggIndexSource(base_url + '/live')})))
app.add_route(
'/upstream_opt',
DefaultResourceHandler(
SimpleAggregator({
'upstream_opt':
UpstreamMementoIndexSource.upstream_resource(base_url +
'/live')
})))
self.base_url = base_url
self.testapp = webtest.TestApp(app)
def load_auto_colls(self):
if not self.root_dir:
print('No Root Dir, Skip Auto Colls!')
return
dir_source = CacheDirectoryIndexSource(base_prefix=self.root_dir,
base_dir=self.index_paths,
config=self.config)
return DefaultResourceHandler(dir_source,
self.archive_paths,
rules_file=self.rules_file)
def setup_class(cls):
super(TestBaseWarcServer, cls).setup_class()
live_source = SimpleAggregator({'live': LiveIndexSource()})
live_handler = DefaultResourceHandler(live_source)
app = BaseWarcServer()
app.add_route('/live', live_handler)
source1 = GeventTimeoutAggregator(sources)
handler1 = DefaultResourceHandler(source1, TEST_WARC_PATH)
app.add_route('/many', handler1)
app.add_route(
'/cdx_api',
DefaultResourceHandler(SimpleAggregator(ia_cdx), TEST_WARC_PATH))
source2 = SimpleAggregator(
{'post': FileIndexSource(TEST_CDX_PATH + 'post-test.cdxj')})
handler2 = DefaultResourceHandler(source2, TEST_WARC_PATH)
app.add_route('/posttest', handler2)
source3 = SimpleAggregator(
{'example': FileIndexSource(TEST_CDX_PATH + 'example2.cdxj')})
handler3 = DefaultResourceHandler(source3, TEST_WARC_PATH)
app.add_route('/fallback',
HandlerSeq([handler3, handler2, live_handler]))
app.add_route('/seq', HandlerSeq([handler3, handler2]))
app.add_route(
'/allredis',
DefaultResourceHandler(source3, 'redis://localhost/2/test:warc'))
app.add_route('/empty', HandlerSeq([]))
app.add_route(
'/invalid',
DefaultResourceHandler(
[SimpleAggregator({'invalid': 'should not be a callable'})]))
url_agnost = SimpleAggregator({
'url-agnost':
FileIndexSource(TEST_CDX_PATH + 'url-agnost-example.cdxj')
})
app.add_route(
'/urlagnost',
DefaultResourceHandler(url_agnost,
'redis://localhost/2/test:{arg}:warc'))
cls.testapp = webtest.TestApp(app)
def load_auto_colls(self):
if not self.root_dir:
print('No Root Dir, Skip Auto Colls!')
return
dir_source = CacheDirectoryIndexSource(base_prefix=self.root_dir,
base_dir=self.index_paths,
config=self.config)
access_checker = AccessChecker(CacheDirectoryAccessSource(self.acl_paths),
self.default_access)
return DefaultResourceHandler(dir_source, self.archive_paths,
rules_file=self.rules_file,
access_checker=access_checker)
def load_coll(self, name, coll_config):
if coll_config == '$all' and self.auto_handler:
return self.auto_handler
if isinstance(coll_config, str):
index = coll_config
archive_paths = None
elif isinstance(coll_config, dict):
index = coll_config.get('index')
if not index:
index = coll_config.get('index_paths')
archive_paths = coll_config.get('archive_paths')
else:
raise Exception('collection config must be string or dict')
if index:
agg = init_index_agg({name: index})
else:
if not isinstance(coll_config, dict):
raise Exception('collection config missing')
sequence = coll_config.get('sequence')
if sequence:
return self.init_sequence(name, sequence)
index_group = coll_config.get('index_group')
if not index_group:
raise Exception('no index, index_group or sequence found')
timeout = int(coll_config.get('timeout', 0))
agg = init_index_agg(index_group, True, timeout)
if not archive_paths:
archive_paths = self.config.get('archive_paths')
return DefaultResourceHandler(agg,
archive_paths,
rules_file=self.rules_file)
def __init__(self):
init_logging()
config = load_wr_config()
app = BaseWarcServer(debug=True)
redis_base = os.environ['REDIS_BASE_URL'] + '/'
rec_url = redis_base + config['cdxj_key_templ']
coll_url = redis_base + config['coll_cdxj_key_templ']
warc_url = redis_base + config['warc_key_templ']
rec_list_key = config['rec_list_key_templ']
redis_resolver = RedisResolver(redis_url=warc_url,
member_key_templ=rec_list_key)
redis = redis_resolver.redis
warc_resolvers = [redis_resolver]
cache_proxy_url = os.environ.get('CACHE_PROXY_URL', '')
global PROXY_PREFIX
PROXY_PREFIX = cache_proxy_url
timeout = 20.0
rec_redis_source = RedisIndexSource(timeout=timeout,
redis_url=rec_url,
redis=redis)
coll_redis_source = RedisIndexSource(timeout=timeout,
redis_url=coll_url,
redis=redis)
live_rec = DefaultResourceHandler(
SimpleAggregator({'live': LiveIndexSource()}, ), warc_resolvers,
cache_proxy_url)
# Extractable archives (all available)
wam_loader = WAMSourceLoader(memento_cls=ProxyMementoIndexSource,
remote_cls=ProxyRemoteIndexSource,
wb_memento_cls=ProxyWBMementoIndexSource)
extractable_archives = wam_loader.sources
# Extract Source
extractor = GeventTimeoutAggregator(extractable_archives,
timeout=timeout)
extract_primary = DefaultResourceHandler(extractor, warc_resolvers,
cache_proxy_url)
# Patch fallback archives
fallback_archives = self.filter_archives(
extractable_archives, config['patch_archives_index'])
# patch + live
#patch_archives = fallback_archives.copy()
patch_archives = fallback_archives
patch_archives['live'] = LiveIndexSource()
extractor2 = GeventTimeoutAggregator(patch_archives,
timeout=timeout,
sources_key='inv_sources',
invert_sources=True)
extract_other = DefaultResourceHandler(extractor2, warc_resolvers,
cache_proxy_url)
patcher = GeventTimeoutAggregator(patch_archives, timeout=timeout)
patch_rec = DefaultResourceHandler(patcher, warc_resolvers,
cache_proxy_url)
# Single Rec Replay
replay_rec = DefaultResourceHandler(
SimpleAggregator({'local': rec_redis_source}), warc_resolvers,
cache_proxy_url)
# Coll Replay
replay_coll = DefaultResourceHandler(
SimpleAggregator({'local': coll_redis_source}), warc_resolvers,
cache_proxy_url)
app.add_route('/live', live_rec)
app.add_route('/extract',
HandlerSeq([extract_primary, extract_other, replay_rec]))
app.add_route('/replay', replay_rec)
app.add_route('/replay-coll', replay_coll)
app.add_route('/patch', HandlerSeq([replay_coll, patch_rec]))
self.app = app