def init(): init_logging(debug=True) config = load_wr_config() wr = WebRecRecorder(config) spawn_once(wr.msg_listen_loop) wr.init_app() wr.app.wr = wr return wr.app
def init(): init_logging() config = load_wr_config() wr = WebRecRecorder(config) spawn_once(wr.msg_listen_loop) wr.init_app() wr.app.wr = wr return wr.app
def init(): init_logging() config = load_wr_config() wr = WebRecRecorder(config) gevent.spawn(wr.msg_listen_loop) wr.init_app(None) wr.app.wr = wr return wr.app
def init(): init_logging(debug=literal_eval(os.environ.get('WR_DEBUG', 'True'))) config = load_wr_config() wr = WebRecRecorder(config) spawn_once(wr.msg_listen_loop) wr.init_app() wr.app.wr = wr return wr.app
def init(): init_logging() config = load_wr_config() wr = WebRecRecorder(config) if postfork: @postfork def listen_loop(): if uwsgi.mule_id() == 0: gevent.spawn(wr.msg_listen_loop) else: gevent.spawn(wr.msg_listen_loop) wr.init_app(None) wr.app.wr = wr return wr.app
def __init__(self): init_logging() config = load_wr_config() app = BaseWarcServer(debug=True) redis_base = os.environ['REDIS_BASE_URL'] + '/' rec_url = redis_base + config['cdxj_key_templ'] coll_url = redis_base + config['coll_cdxj_key_templ'] warc_url = redis_base + config['warc_key_templ'] rec_list_key = config['rec_list_key_templ'] redis_resolver = RedisResolver(redis_url=warc_url, member_key_templ=rec_list_key) redis = redis_resolver.redis warc_resolvers = [redis_resolver] cache_proxy_url = os.environ.get('CACHE_PROXY_URL', '') global PROXY_PREFIX PROXY_PREFIX = cache_proxy_url timeout = 20.0 rec_redis_source = RedisIndexSource(timeout=timeout, redis_url=rec_url, redis=redis) coll_redis_source = RedisIndexSource(timeout=timeout, redis_url=coll_url, redis=redis) live_rec = DefaultResourceHandler( SimpleAggregator({'live': LiveIndexSource()}, ), warc_resolvers, cache_proxy_url) # Extractable archives (all available) wam_loader = WAMSourceLoader(memento_cls=ProxyMementoIndexSource, remote_cls=ProxyRemoteIndexSource, wb_memento_cls=ProxyWBMementoIndexSource) extractable_archives = wam_loader.sources # Extract Source extractor = GeventTimeoutAggregator(extractable_archives, timeout=timeout) extract_primary = DefaultResourceHandler(extractor, warc_resolvers, cache_proxy_url) # Patch fallback archives fallback_archives = self.filter_archives( extractable_archives, config['patch_archives_index']) # patch + live #patch_archives = fallback_archives.copy() patch_archives = fallback_archives patch_archives['live'] = LiveIndexSource() extractor2 = GeventTimeoutAggregator(patch_archives, timeout=timeout, sources_key='inv_sources', invert_sources=True) extract_other = DefaultResourceHandler(extractor2, warc_resolvers, cache_proxy_url) patcher = GeventTimeoutAggregator(patch_archives, timeout=timeout) patch_rec = DefaultResourceHandler(patcher, warc_resolvers, cache_proxy_url) # Single Rec Replay replay_rec = DefaultResourceHandler( SimpleAggregator({'local': rec_redis_source}), warc_resolvers, cache_proxy_url) # Coll Replay replay_coll = DefaultResourceHandler( SimpleAggregator({'local': coll_redis_source}), warc_resolvers, cache_proxy_url) app.add_route('/live', live_rec) app.add_route('/extract', HandlerSeq([extract_primary, extract_other, replay_rec])) app.add_route('/replay', replay_rec) app.add_route('/replay-coll', replay_coll) app.add_route('/patch', HandlerSeq([replay_coll, patch_rec])) self.app = app
def _init_logging(self): # bottle debug debug(True) init_logging()
def __init__(self): init_logging() config = load_wr_config() app = BaseWarcServer(debug=True) redis_base = os.environ['REDIS_BASE_URL'] + '/' #rec_url = redis_base + config['cdxj_key_templ'] #coll_url = redis_base + config['coll_cdxj_key_templ'] #warc_url = redis_base + config['coll_warc_key_templ'] #rec_map_key = config['rec_map_key_templ'] rec_url = redis_base + Recording.CDXJ_KEY coll_url = redis_base + Collection.COLL_CDXJ_KEY warc_url = redis_base + Recording.COLL_WARC_KEY rec_map_key = Collection.RECS_KEY redis_resolver = RedisResolver(redis_url=warc_url, member_key_templ=rec_map_key) redis = redis_resolver.redis warc_resolvers = [redis_resolver] cache_proxy_url = os.environ.get('CACHE_PROXY_URL', '') global PROXY_PREFIX PROXY_PREFIX = cache_proxy_url timeout = 20.0 rec_redis_source = RedisIndexSource(timeout=timeout, redis_url=rec_url, redis=redis) coll_redis_source = RedisIndexSource(timeout=timeout, redis_url=coll_url, redis=redis) live_rec = DefaultResourceHandler( SimpleAggregator( {'live': LiveIndexSource()}, ), warc_resolvers, cache_proxy_url) # Extractable archives (all available) wam_loader = WAMSourceLoader(memento_cls=ProxyMementoIndexSource, remote_cls=ProxyRemoteIndexSource, wb_memento_cls=ProxyWBMementoIndexSource) extractable_archives = wam_loader.sources # Extract Source extractor = GeventTimeoutAggregator(extractable_archives, timeout=timeout) extract_primary = DefaultResourceHandler( extractor, warc_resolvers, cache_proxy_url) # Patch fallback archives fallback_archives = self.filter_archives(extractable_archives, config['patch_archives_index']) # patch + live #patch_archives = fallback_archives.copy() patch_archives = fallback_archives patch_archives['live'] = LiveIndexSource() extractor2 = GeventTimeoutAggregator(patch_archives, timeout=timeout, sources_key='inv_sources', invert_sources=True) extract_other = DefaultResourceHandler( extractor2, warc_resolvers, cache_proxy_url) patcher = GeventTimeoutAggregator(patch_archives, timeout=timeout) patch_rec = DefaultResourceHandler( patcher, warc_resolvers, cache_proxy_url) # Single Rec Replay replay_rec = DefaultResourceHandler(SimpleAggregator({'local': rec_redis_source}), warc_resolvers, cache_proxy_url) # Coll Replay replay_coll = DefaultResourceHandler(SimpleAggregator({'local': coll_redis_source}), warc_resolvers, cache_proxy_url) app.add_route('/live', live_rec) app.add_route('/extract', HandlerSeq([extract_primary, extract_other, replay_rec])) app.add_route('/replay', replay_rec) app.add_route('/replay-coll', replay_coll) app.add_route('/patch', HandlerSeq([replay_coll, patch_rec])) self.app = app