def make_live_app(): app = BaseWarcServer() app.add_route( '/live', DefaultResourceHandler( SimpleAggregator({'live': LiveIndexSource()}))) return app
def load_auto_colls(self): if not self.root_dir: print('No Root Dir, Skip Auto Colls!') return dir_source = CacheDirectoryIndexSource(base_prefix=self.root_dir, base_dir=self.index_paths, config=self.config) access_checker = AccessChecker( CacheDirectoryAccessSource(self.acl_paths), self.default_access) if self.dedup_index_url: source = SimpleAggregator({ 'dedup': RedisMultiKeyIndexSource(self.dedup_index_url), 'dir': dir_source }) else: source = dir_source return DefaultResourceHandler(source, self.archive_paths, rules_file=self.rules_file, access_checker=access_checker)
def load_coll(self, name, coll_config): if coll_config == '$all' and self.auto_handler: return self.auto_handler if isinstance(coll_config, str): index = coll_config archive_paths = None acl_paths = None default_access = self.default_access elif isinstance(coll_config, dict): index = coll_config.get('index') if not index: index = coll_config.get('index_paths') archive_paths = coll_config.get('archive_paths') acl_paths = coll_config.get('acl_paths') default_access = coll_config.get('default_access', self.default_access) surt_ordered = coll_config.get('surt_ordered', True) else: raise Exception('collection config must be string or dict') # INDEX CONFIG if index: agg = init_index_agg({name: index}, surt_ordered=surt_ordered) else: if not isinstance(coll_config, dict): raise Exception('collection config missing') sequence = coll_config.get('sequence') if sequence: return self.init_sequence(name, sequence) index_group = coll_config.get('index_group') if not index_group: raise Exception('no index, index_group or sequence found') timeout = int(coll_config.get('timeout', 0)) agg = init_index_agg(index_group, True, timeout, surt_ordered=surt_ordered) # ARCHIVE CONFIG if not archive_paths: archive_paths = self.config.get('archive_paths') # ACCESS CONFIG access_checker = None if acl_paths: access_checker = AccessChecker(acl_paths, default_access) return DefaultResourceHandler(agg, archive_paths, rules_file=self.rules_file, access_checker=access_checker)
def setup(self): app = BaseWarcServer() base_url = 'http://localhost:{0}'.format(self.server.port) app.add_route( '/upstream', DefaultResourceHandler( SimpleAggregator( {'upstream': UpstreamAggIndexSource(base_url + '/live')}))) app.add_route( '/upstream_opt', DefaultResourceHandler( SimpleAggregator({ 'upstream_opt': UpstreamMementoIndexSource.upstream_resource(base_url + '/live') }))) self.base_url = base_url self.testapp = webtest.TestApp(app)
def load_auto_colls(self): if not self.root_dir: print('No Root Dir, Skip Auto Colls!') return dir_source = CacheDirectoryIndexSource(base_prefix=self.root_dir, base_dir=self.index_paths, config=self.config) return DefaultResourceHandler(dir_source, self.archive_paths, rules_file=self.rules_file)
def setup_class(cls): super(TestBaseWarcServer, cls).setup_class() live_source = SimpleAggregator({'live': LiveIndexSource()}) live_handler = DefaultResourceHandler(live_source) app = BaseWarcServer() app.add_route('/live', live_handler) source1 = GeventTimeoutAggregator(sources) handler1 = DefaultResourceHandler(source1, TEST_WARC_PATH) app.add_route('/many', handler1) app.add_route( '/cdx_api', DefaultResourceHandler(SimpleAggregator(ia_cdx), TEST_WARC_PATH)) source2 = SimpleAggregator( {'post': FileIndexSource(TEST_CDX_PATH + 'post-test.cdxj')}) handler2 = DefaultResourceHandler(source2, TEST_WARC_PATH) app.add_route('/posttest', handler2) source3 = SimpleAggregator( {'example': FileIndexSource(TEST_CDX_PATH + 'example2.cdxj')}) handler3 = DefaultResourceHandler(source3, TEST_WARC_PATH) app.add_route('/fallback', HandlerSeq([handler3, handler2, live_handler])) app.add_route('/seq', HandlerSeq([handler3, handler2])) app.add_route( '/allredis', DefaultResourceHandler(source3, 'redis://localhost/2/test:warc')) app.add_route('/empty', HandlerSeq([])) app.add_route( '/invalid', DefaultResourceHandler( [SimpleAggregator({'invalid': 'should not be a callable'})])) url_agnost = SimpleAggregator({ 'url-agnost': FileIndexSource(TEST_CDX_PATH + 'url-agnost-example.cdxj') }) app.add_route( '/urlagnost', DefaultResourceHandler(url_agnost, 'redis://localhost/2/test:{arg}:warc')) cls.testapp = webtest.TestApp(app)
def load_auto_colls(self): if not self.root_dir: print('No Root Dir, Skip Auto Colls!') return dir_source = CacheDirectoryIndexSource(base_prefix=self.root_dir, base_dir=self.index_paths, config=self.config) access_checker = AccessChecker(CacheDirectoryAccessSource(self.acl_paths), self.default_access) return DefaultResourceHandler(dir_source, self.archive_paths, rules_file=self.rules_file, access_checker=access_checker)
def load_coll(self, name, coll_config): if coll_config == '$all' and self.auto_handler: return self.auto_handler if isinstance(coll_config, str): index = coll_config archive_paths = None elif isinstance(coll_config, dict): index = coll_config.get('index') if not index: index = coll_config.get('index_paths') archive_paths = coll_config.get('archive_paths') else: raise Exception('collection config must be string or dict') if index: agg = init_index_agg({name: index}) else: if not isinstance(coll_config, dict): raise Exception('collection config missing') sequence = coll_config.get('sequence') if sequence: return self.init_sequence(name, sequence) index_group = coll_config.get('index_group') if not index_group: raise Exception('no index, index_group or sequence found') timeout = int(coll_config.get('timeout', 0)) agg = init_index_agg(index_group, True, timeout) if not archive_paths: archive_paths = self.config.get('archive_paths') return DefaultResourceHandler(agg, archive_paths, rules_file=self.rules_file)
def __init__(self): init_logging() config = load_wr_config() app = BaseWarcServer(debug=True) redis_base = os.environ['REDIS_BASE_URL'] + '/' rec_url = redis_base + config['cdxj_key_templ'] coll_url = redis_base + config['coll_cdxj_key_templ'] warc_url = redis_base + config['warc_key_templ'] rec_list_key = config['rec_list_key_templ'] redis_resolver = RedisResolver(redis_url=warc_url, member_key_templ=rec_list_key) redis = redis_resolver.redis warc_resolvers = [redis_resolver] cache_proxy_url = os.environ.get('CACHE_PROXY_URL', '') global PROXY_PREFIX PROXY_PREFIX = cache_proxy_url timeout = 20.0 rec_redis_source = RedisIndexSource(timeout=timeout, redis_url=rec_url, redis=redis) coll_redis_source = RedisIndexSource(timeout=timeout, redis_url=coll_url, redis=redis) live_rec = DefaultResourceHandler( SimpleAggregator({'live': LiveIndexSource()}, ), warc_resolvers, cache_proxy_url) # Extractable archives (all available) wam_loader = WAMSourceLoader(memento_cls=ProxyMementoIndexSource, remote_cls=ProxyRemoteIndexSource, wb_memento_cls=ProxyWBMementoIndexSource) extractable_archives = wam_loader.sources # Extract Source extractor = GeventTimeoutAggregator(extractable_archives, timeout=timeout) extract_primary = DefaultResourceHandler(extractor, warc_resolvers, cache_proxy_url) # Patch fallback archives fallback_archives = self.filter_archives( extractable_archives, config['patch_archives_index']) # patch + live #patch_archives = fallback_archives.copy() patch_archives = fallback_archives patch_archives['live'] = LiveIndexSource() extractor2 = GeventTimeoutAggregator(patch_archives, timeout=timeout, sources_key='inv_sources', invert_sources=True) extract_other = DefaultResourceHandler(extractor2, warc_resolvers, cache_proxy_url) patcher = GeventTimeoutAggregator(patch_archives, timeout=timeout) patch_rec = DefaultResourceHandler(patcher, warc_resolvers, cache_proxy_url) # Single Rec Replay replay_rec = DefaultResourceHandler( SimpleAggregator({'local': rec_redis_source}), warc_resolvers, cache_proxy_url) # Coll Replay replay_coll = DefaultResourceHandler( SimpleAggregator({'local': coll_redis_source}), warc_resolvers, cache_proxy_url) app.add_route('/live', live_rec) app.add_route('/extract', HandlerSeq([extract_primary, extract_other, replay_rec])) app.add_route('/replay', replay_rec) app.add_route('/replay-coll', replay_coll) app.add_route('/patch', HandlerSeq([replay_coll, patch_rec])) self.app = app