def init():
init_logging(debug=True)
config = load_wr_config()
wr = WebRecRecorder(config)
spawn_once(wr.msg_listen_loop)
wr.init_app()
wr.app.wr = wr
return wr.app
def init():
init_logging()
config = load_wr_config()
wr = WebRecRecorder(config)
spawn_once(wr.msg_listen_loop)
wr.init_app()
wr.app.wr = wr
return wr.app
def init():
init_logging()
config = load_wr_config()
wr = WebRecRecorder(config)
gevent.spawn(wr.msg_listen_loop)
wr.init_app(None)
wr.app.wr = wr
return wr.app
def init():
init_logging(debug=literal_eval(os.environ.get('WR_DEBUG', 'True')))
config = load_wr_config()
wr = WebRecRecorder(config)
spawn_once(wr.msg_listen_loop)
wr.init_app()
wr.app.wr = wr
return wr.app
def init():
init_logging()
config = load_wr_config()
wr = WebRecRecorder(config)
if postfork:
@postfork
def listen_loop():
if uwsgi.mule_id() == 0:
gevent.spawn(wr.msg_listen_loop)
else:
gevent.spawn(wr.msg_listen_loop)
wr.init_app(None)
wr.app.wr = wr
return wr.app
def __init__(self):
init_logging()
config = load_wr_config()
app = BaseWarcServer(debug=True)
redis_base = os.environ['REDIS_BASE_URL'] + '/'
rec_url = redis_base + config['cdxj_key_templ']
coll_url = redis_base + config['coll_cdxj_key_templ']
warc_url = redis_base + config['warc_key_templ']
rec_list_key = config['rec_list_key_templ']
redis_resolver = RedisResolver(redis_url=warc_url,
member_key_templ=rec_list_key)
redis = redis_resolver.redis
warc_resolvers = [redis_resolver]
cache_proxy_url = os.environ.get('CACHE_PROXY_URL', '')
global PROXY_PREFIX
PROXY_PREFIX = cache_proxy_url
timeout = 20.0
rec_redis_source = RedisIndexSource(timeout=timeout,
redis_url=rec_url,
redis=redis)
coll_redis_source = RedisIndexSource(timeout=timeout,
redis_url=coll_url,
redis=redis)
live_rec = DefaultResourceHandler(
SimpleAggregator({'live': LiveIndexSource()}, ), warc_resolvers,
cache_proxy_url)
# Extractable archives (all available)
wam_loader = WAMSourceLoader(memento_cls=ProxyMementoIndexSource,
remote_cls=ProxyRemoteIndexSource,
wb_memento_cls=ProxyWBMementoIndexSource)
extractable_archives = wam_loader.sources
# Extract Source
extractor = GeventTimeoutAggregator(extractable_archives,
timeout=timeout)
extract_primary = DefaultResourceHandler(extractor, warc_resolvers,
cache_proxy_url)
# Patch fallback archives
fallback_archives = self.filter_archives(
extractable_archives, config['patch_archives_index'])
# patch + live
#patch_archives = fallback_archives.copy()
patch_archives = fallback_archives
patch_archives['live'] = LiveIndexSource()
extractor2 = GeventTimeoutAggregator(patch_archives,
timeout=timeout,
sources_key='inv_sources',
invert_sources=True)
extract_other = DefaultResourceHandler(extractor2, warc_resolvers,
cache_proxy_url)
patcher = GeventTimeoutAggregator(patch_archives, timeout=timeout)
patch_rec = DefaultResourceHandler(patcher, warc_resolvers,
cache_proxy_url)
# Single Rec Replay
replay_rec = DefaultResourceHandler(
SimpleAggregator({'local': rec_redis_source}), warc_resolvers,
cache_proxy_url)
# Coll Replay
replay_coll = DefaultResourceHandler(
SimpleAggregator({'local': coll_redis_source}), warc_resolvers,
cache_proxy_url)
app.add_route('/live', live_rec)
app.add_route('/extract',
HandlerSeq([extract_primary, extract_other, replay_rec]))
app.add_route('/replay', replay_rec)
app.add_route('/replay-coll', replay_coll)
app.add_route('/patch', HandlerSeq([replay_coll, patch_rec]))
self.app = app
def _init_logging(self):
# bottle debug
debug(True)
init_logging()
def __init__(self):
init_logging()
config = load_wr_config()
app = BaseWarcServer(debug=True)
redis_base = os.environ['REDIS_BASE_URL'] + '/'
#rec_url = redis_base + config['cdxj_key_templ']
#coll_url = redis_base + config['coll_cdxj_key_templ']
#warc_url = redis_base + config['coll_warc_key_templ']
#rec_map_key = config['rec_map_key_templ']
rec_url = redis_base + Recording.CDXJ_KEY
coll_url = redis_base + Collection.COLL_CDXJ_KEY
warc_url = redis_base + Recording.COLL_WARC_KEY
rec_map_key = Collection.RECS_KEY
redis_resolver = RedisResolver(redis_url=warc_url,
member_key_templ=rec_map_key)
redis = redis_resolver.redis
warc_resolvers = [redis_resolver]
cache_proxy_url = os.environ.get('CACHE_PROXY_URL', '')
global PROXY_PREFIX
PROXY_PREFIX = cache_proxy_url
timeout = 20.0
rec_redis_source = RedisIndexSource(timeout=timeout,
redis_url=rec_url,
redis=redis)
coll_redis_source = RedisIndexSource(timeout=timeout,
redis_url=coll_url,
redis=redis)
live_rec = DefaultResourceHandler(
SimpleAggregator(
{'live': LiveIndexSource()},
), warc_resolvers,
cache_proxy_url)
# Extractable archives (all available)
wam_loader = WAMSourceLoader(memento_cls=ProxyMementoIndexSource,
remote_cls=ProxyRemoteIndexSource,
wb_memento_cls=ProxyWBMementoIndexSource)
extractable_archives = wam_loader.sources
# Extract Source
extractor = GeventTimeoutAggregator(extractable_archives, timeout=timeout)
extract_primary = DefaultResourceHandler(
extractor,
warc_resolvers,
cache_proxy_url)
# Patch fallback archives
fallback_archives = self.filter_archives(extractable_archives,
config['patch_archives_index'])
# patch + live
#patch_archives = fallback_archives.copy()
patch_archives = fallback_archives
patch_archives['live'] = LiveIndexSource()
extractor2 = GeventTimeoutAggregator(patch_archives, timeout=timeout,
sources_key='inv_sources',
invert_sources=True)
extract_other = DefaultResourceHandler(
extractor2,
warc_resolvers,
cache_proxy_url)
patcher = GeventTimeoutAggregator(patch_archives, timeout=timeout)
patch_rec = DefaultResourceHandler(
patcher,
warc_resolvers,
cache_proxy_url)
# Single Rec Replay
replay_rec = DefaultResourceHandler(SimpleAggregator({'local': rec_redis_source}),
warc_resolvers,
cache_proxy_url)
# Coll Replay
replay_coll = DefaultResourceHandler(SimpleAggregator({'local': coll_redis_source}),
warc_resolvers,
cache_proxy_url)
app.add_route('/live', live_rec)
app.add_route('/extract', HandlerSeq([extract_primary, extract_other, replay_rec]))
app.add_route('/replay', replay_rec)
app.add_route('/replay-coll', replay_coll)
app.add_route('/patch', HandlerSeq([replay_coll, patch_rec]))
self.app = app