def __init__(self, config_file=None, custom_config=None): """ :param str|None config_file: Path to the config file :param dict|None custom_config: Dictionary containing additional configuration information """ config_file = config_file or './config.yaml' self.handler = self.handle_request self.warcserver = WarcServer(config_file=config_file, custom_config=custom_config) self.recorder = None self.recorder_path = None self.put_custom_record_path = None self.proxy_default_timestamp = None config = self.warcserver.config self.debug = config.get('debug', False) self.warcserver_server = GeventServer(self.warcserver, port=0) self.proxy_prefix = None # the URL prefix to be used for the collection with proxy mode (e.g. /coll/id_/) self.proxy_coll = None # the name of the collection that has proxy mode enabled self.proxy_record = False # indicate if proxy recording self.init_proxy(config) self.init_recorder(config.get('recorder')) self.init_autoindex(config.get('autoindex')) static_path = config.get('static_url_path', 'pywb/static/').replace('/', os.path.sep) self.static_handler = StaticHandler(static_path) self.cdx_api_endpoint = config.get('cdx_api_endpoint', '/cdx') self.query_limit = config.get('query_limit') upstream_paths = self.get_upstream_paths(self.warcserver_server.port) framed_replay = config.get('framed_replay', True) self.rewriterapp = self.REWRITER_APP_CLS(framed_replay, config=config, paths=upstream_paths) self.templates_dir = config.get('templates_dir', 'templates') self.static_dir = config.get('static_dir', 'static') metadata_templ = os.path.join(self.warcserver.root_dir, '{coll}', 'metadata.yaml') self.metadata_cache = MetadataCache(metadata_templ) self._init_routes()
def setup_class(cls): super(TestWarcServer, cls).setup_class() cls.orig_cwd = os.getcwd() os.chdir(cls.root_dir) os.mkdir('./local') os.mkdir('./local/indexes') os.mkdir('collections') os.mkdir('collections/auto1') os.mkdir('collections/auto2') with open(os.path.join('local', 'indexes', 'file.cdxj'), 'a') as fh: fh.write('foo') with open(os.path.join('local', 'indexes', 'file.idx'), 'a') as fh: fh.write('foo') with open(os.path.join('local', 'indexes', 'file.loc'), 'a') as fh: fh.write('foo') cls.loader = WarcServer( os.path.join(cls.get_curr_dir(), 'test_warcserver_config.yaml'))
def __init__(self, config_file='./config.yaml', custom_config=None): self.handler = self.handle_request self.warcserver = WarcServer(config_file=config_file, custom_config=custom_config) config = self.warcserver.config self.debug = config.get('debug', False) self.warcserver_server = GeventServer(self.warcserver, port=0) self.init_proxy(config) self.init_recorder(config.get('recorder')) self.init_autoindex(config.get('autoindex')) static_path = config.get('static_url_path', 'pywb/static/').replace('/', os.path.sep) self.static_handler = StaticHandler(static_path) self.cdx_api_endpoint = config.get('cdx_api_endpoint', '/cdx') self._init_routes() upstream_paths = self.get_upstream_paths(self.warcserver_server.port) framed_replay = config.get('framed_replay', True) self.rewriterapp = RewriterApp(framed_replay, config=config, paths=upstream_paths) self.templates_dir = config.get('templates_dir', 'templates') self.static_dir = config.get('static_dir', 'static') metadata_templ = os.path.join(self.warcserver.root_dir, '{coll}', 'metadata.yaml') self.metadata_cache = MetadataCache(metadata_templ)
def setup_class(cls): super(TestCDXApp, cls).setup_class() config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config_test.yaml') cls.testapp = webtest.TestApp(WarcServer(config_file=config_file))
def load(self): from pywb.warcserver.warcserver import WarcServer super(WarcServerCli, self).load() return WarcServer(custom_config=self.extra_config)
from gevent.monkey import patch_all patch_all() from pywb.warcserver.warcserver import WarcServer application = WarcServer(custom_config={'collections': {'live': '$live'}})