def setup_module():
global requestlog
requestlog = []
def make_httpd(app):
global proxyserv
proxyserv = ProxyServer(('', 0), ProxyRequest)
proxyserv.requestlog = requestlog
proxyserv.force_err = False
return proxyserv
global server
server = ServerThreadRunner(make_httpd)
config = dict(collections=dict(rewrite='$liveweb'),
framed_replay=True,
proxyhostport=server.proxy_str)
global cache
cache = {}
def create_cache():
return cache
pywb.webapp.live_rewrite_handler.create_cache = create_cache
global app
app = init_app(create_wb_router, load_yaml=False, config=config)
global testapp
testapp = webtest.TestApp(app)
def __init__(self, archivefiles):
self.archivefiles = archivefiles
self.coll_name = ''
self.cdx_file = tempfile.NamedTemporaryFile(delete=False,
suffix='.cdxj',
prefix='cdx')
self.path_index = tempfile.NamedTemporaryFile(delete=False,
suffix='.txt')
try:
pagelist = self.update_cdx(self.cdx_file, archivefiles)
except Exception as e:
msg = "WebArchivePlayer is unable to read the input file(s) and will quit.\n\nDetails: " + str(e)[:255]
if no_wx:
sys.stderr.write(msg + '\n')
else:
dlg = wx.MessageDialog(None, msg, "Error Reading Web Archive File(s)", style=wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
sys.exit(1)
config = self._load_dynamic_config()
config['_pagelist'] = pagelist
config['_archivefiles'] = archivefiles
self.write_path_index()
self.application = init_app(create_wb_router,
load_yaml=False,
config=config)
def setup(self):
self.requestlog = []
self.cache = {}
def make_httpd(app):
proxyserv = ProxyServer(('', 0), ProxyRequest)
proxyserv.requestlog = self.requestlog
proxyserv.force_err = False
self.proxyserv = proxyserv
return proxyserv
self.server = ServerThreadRunner(make_httpd)
config = dict(collections=dict(rewrite='$liveweb'),
framed_replay=True,
proxyhostport=self.server.proxy_dict)
self.app = init_app(create_wb_router, load_yaml=False, config=config)
def create_cache():
return self.cache
pywb.webapp.live_rewrite_handler.create_cache = create_cache
self.testapp = webtest.TestApp(self.app)
def load(self):
config = dict(proxyhostport=self.r.proxy,
framed_replay='inverse' if self.r.framed else False,
enable_auto_colls=False,
collections={'live': '$liveweb'})
return init_app(create_wb_router, load_yaml=False, config=config)
def __init__(self, archivefiles):
self.archivefiles = archivefiles
self.coll_name = ''
self.cdx_file = tempfile.NamedTemporaryFile(delete=False,
suffix='.cdxj',
prefix='cdx')
self.path_index = tempfile.NamedTemporaryFile(delete=False,
suffix='.txt')
try:
pagelist = self.update_cdx(self.cdx_file, archivefiles)
except Exception as e:
msg = "WebArchivePlayer is unable to read the input file(s) and will quit.\n\nDetails: " + str(
e)[:255]
if no_wx:
sys.stderr.write(msg + '\n')
else:
dlg = wx.MessageDialog(None,
msg,
"Error Reading Web Archive File(s)",
style=wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
sys.exit(1)
config = self._load_dynamic_config()
config['_pagelist'] = pagelist
config['_archivefiles'] = archivefiles
self.write_path_index()
self.application = init_app(create_wb_router,
load_yaml=False,
config=config)
def load(self):
config = dict(proxyhostport=self.r.proxy,
framed_replay='inverse' if self.r.framed else False,
enable_auto_colls=False,
collections={'live': '$liveweb'})
return init_app(create_wb_router, load_yaml=False, config=config)
def setup_module():
global requestlog
requestlog = []
def make_httpd(app):
global proxyserv
proxyserv = ProxyServer(('', 0), ProxyRequest)
proxyserv.requestlog = requestlog
proxyserv.force_err = False
return proxyserv
global server
server = ServerThreadRunner(make_httpd)
config = dict(collections=dict(rewrite='$liveweb'),
framed_replay=True,
proxyhostport=server.proxy_dict)
global cache
cache = {}
def create_cache():
return cache
pywb.webapp.live_rewrite_handler.create_cache = create_cache
global app
app = init_app(create_wb_router,
load_yaml=False,
config=config)
global testapp
testapp = webtest.TestApp(app)
def make_app(config_file, pywb_router=create_wb_router):
app = init_app(pywb_router,
load_yaml=True,
config_file=config_file)
testapp = TestApp(app)
return app, testapp
def test_ok_app():
the_app = init_app(initer(TestOkApp), load_yaml=False)
testapp = webtest.TestApp(the_app)
resp = testapp.get('/')
assert resp.status_int == 200
assert 'Test' in resp.body
def test_custom_err_app():
the_app = init_app(initer(TestCustomErrApp), load_yaml=False)
testapp = webtest.TestApp(the_app)
resp = testapp.get('/abc', expect_errors=True)
assert resp.status_int == 403
assert '403 Access Denied Error: Forbidden Test' in resp.body
def test_err_app():
the_app = init_app(initer(TestErrApp), load_yaml=False)
testapp = webtest.TestApp(the_app)
resp = testapp.get('/abc', expect_errors=True)
assert resp.status_int == 500
assert '500 Internal Server Error Error: Test Unexpected Error' in resp.body
def test_err_app():
the_app = init_app(initer(TestErrApp), load_yaml=False)
testapp = webtest.TestApp(the_app)
resp = testapp.get('/abc', expect_errors=True)
assert resp.status_int == 400
assert '400 Bad Request Error: Test Error' in resp.body
def test_err_app():
the_app = init_app(initer(TestErrApp), load_yaml=False)
testapp = webtest.TestApp(the_app)
resp = testapp.get('/abc', expect_errors=True)
assert resp.status_int == 400
assert '400 Bad Request Error: Test Error' in resp.body
def setup(self):
#self.app = pywb.wbapp.create_wb_app(pywb.pywb_init.pywb_config())
# save it in self - useful for debugging
self.app = init_app(create_wb_router,
load_yaml=True,
config_file=self.TEST_CONFIG)
#self.router = pywb_config(self.TEST_CONFIG)
#self.app = create_wb_app(self.router)
self.testapp = webtest.TestApp(self.app)
def setup(self):
#self.app = pywb.wbapp.create_wb_app(pywb.pywb_init.pywb_config())
# save it in self - useful for debugging
self.app = init_app(create_wb_router,
load_yaml=True,
config_file=self.TEST_CONFIG)
#self.router = pywb_config(self.TEST_CONFIG)
#self.app = create_wb_app(self.router)
self.testapp = webtest.TestApp(self.app)
def __init__(self, *args, **kwargs):
super(ServeThread, self).__init__(*args, **kwargs)
self.app = init_app(create_wb_router,
load_yaml=True,
config_file=TEST_CONFIG)
# init with port 0 to allow os to pick a port
self.httpd = make_server('', 0, self.app)
port = self.httpd.socket.getsockname()[1]
proxy_str = 'http://localhost:' + str(port)
self.proxy_dict = {'http': proxy_str}
def make_app(config_file, pywb_router=create_wb_router):
app = init_app(pywb_router, load_yaml=True, config_file=config_file)
testapp = TestApp(app)
class Resp(TestResponse):
def __init__(self, *args, **kwargs):
super(Resp, self).__init__(*args, **kwargs)
if self.headers.get('Content-Type'):
self.charset = 'utf-8'
TestApp.RequestClass.ResponseClass = Resp
return app, testapp
def make_app(config_file, pywb_router=create_wb_router):
app = init_app(pywb_router, load_yaml=True, config_file=config_file)
testapp = TestApp(app)
class Resp(TestResponse):
def __init__(self, *args, **kwargs):
super(Resp, self).__init__(*args, **kwargs)
if self.headers.get("Content-Type"):
self.charset = "utf-8"
TestApp.RequestClass.ResponseClass = Resp
return app, testapp
def create_app():
parser = ArgumentParser(description='Live Rewrite Server')
parser.add_argument('-x', '--proxy',
action='store',
help='Specify host:port to use as HTTP/S proxy')
result, unknown = parser.parse_known_args()
config = dict(proxyhostport=result.proxy, framed_replay=True)
app = init_app(create_live_rewriter_app, load_yaml=False,
config=config)
return app
def __init__(self, make_httpd, config_file=None):
if config_file:
self.app = init_app(create_wb_router,
load_yaml=True,
config_file=config_file)
else:
self.app = None
self.httpd = make_httpd(self.app)
self.port = self.httpd.socket.getsockname()[1]
proxy_str = 'http://localhost:' + str(self.port)
self.proxy_dict = {'http': proxy_str,
'https': proxy_str}
def run():
self.httpd.serve_forever()
self.thread = threading.Thread(target=run)
self.thread.daemon = True
self.thread.start()
def __init__(self, archivefiles):
self.archivefiles = archivefiles
self.coll_name = ''
self.cdx_file = tempfile.NamedTemporaryFile(delete=False,
suffix='.cdxj',
prefix='cdx')
self.path_index = tempfile.NamedTemporaryFile(delete=False,
suffix='.txt')
pagelist = self.update_cdx(self.cdx_file, archivefiles)
config = self._load_dynamic_config()
config['_pagelist'] = pagelist
config['_archivefiles'] = archivefiles
self.write_path_index()
self.application = init_app(create_wb_router,
load_yaml=False,
config=config)
def __init__(self, archivefiles):
self.archivefiles = archivefiles
self.coll_name = ''
self.cdx_file = tempfile.NamedTemporaryFile(delete=False,
suffix='.cdxj',
prefix='cdx')
self.path_index = tempfile.NamedTemporaryFile(delete=False,
suffix='.txt')
pagelist = self.update_cdx(self.cdx_file, archivefiles)
config = self._load_config()
config['_pagelist'] = pagelist
config['_archivefiles'] = archivefiles
self.write_path_index()
self.application = init_app(create_wb_router,
load_yaml=False,
config=config)
def setup(self):
self.requestlog = []
self.cache = {}
def make_httpd(app):
proxyserv = ProxyServer(("", 0), ProxyRequest)
proxyserv.requestlog = self.requestlog
proxyserv.force_err = False
self.proxyserv = proxyserv
return proxyserv
self.server = ServerThreadRunner(make_httpd)
config = dict(collections=dict(rewrite="$liveweb"), framed_replay=True, proxyhostport=self.server.proxy_dict)
self.app = init_app(create_wb_router, load_yaml=False, config=config)
def create_cache():
return self.cache
pywb.webapp.live_rewrite_handler.create_cache = create_cache
self.testapp = webtest.TestApp(self.app)
def _create_app(self):
self.app = init_app(create_wb_router)
self.testapp = webtest.TestApp(self.app)
from pywb.framework.wsgi_wrappers import init_app from pywb.webapp.pywb_init import create_wb_router #================================================================= # init pywb app #================================================================= application = init_app(create_wb_router, load_yaml=True)
def __init__(self, static_config):
self.application = init_app(create_wb_router,
load_yaml=False,
config=static_config)
def setup(self):
self.app = init_app(create_live_rewriter_app,
load_yaml=False,
config=dict(framed_replay=True))
self.testapp = webtest.TestApp(self.app)
from pywb.framework.wsgi_wrappers import init_app, start_wsgi_server
#from pywb.core.cdx_api_handler import create_cdx_server_app
from pywb.webapp.pywb_init import create_cdx_server_app
#=================================================================
# init cdx server app
#=================================================================
application = init_app(create_cdx_server_app, load_yaml=True)
def main(): # pragma: no cover
start_wsgi_server(application, 'CDX Server', default_port=8090)
if __name__ == "__main__":
main()
def setup(self):
self.app = init_app(create_wb_router,
load_yaml=True,
config_file=self.TEST_CONFIG)
self.testapp = webtest.TestApp(self.app)
def load(self):
super(WaybackCli, self).load()
return init_app(create_wb_router, load_yaml=True)
def load(self):
super(CdxCli, self).load()
return init_app(create_cdx_server_app, load_yaml=True)
def setup(self):
self.app = init_app(create_live_rewriter_app, load_yaml=False,
config=dict(framed_replay=True))
self.testapp = webtest.TestApp(self.app)
def load(self):
super(WaybackCli, self).load()
return init_app(create_wb_router,
load_yaml=True)
def __init__(self, static_config):
self.application = init_app(create_wb_router,
load_yaml=False,
config=static_config)
def setup(self):
self.app = init_app(create_perms_checker_app,
load_yaml=True,
config_file=self.TEST_CONFIG)
self.testapp = webtest.TestApp(self.app)
except:
extra = {}
logging.error(traceback.format_exc(exc), extra=extra)
return real_handle_exception(self, env, exc, print_trace)
WSGIApp.handle_exception = handle_exception
application = init_app(
create_perma_wb_router,
load_yaml=False,
config={
'port': 8000,
'collections': {
'': 'PermaCDXSource'
},
'archive_paths': get_archive_path(),
'server_cls': PermaCDXServer,
'wb_handler_class': PermaHandler,
'enable_memento': True,
'framed_replay': False,
# pywb template vars (used in templates called by pywb, such as head_insert.html, but not our ErrorTemplateView)
'template_globals': {
'static_path': settings.STATIC_URL.rstrip('/') + '/pywb'
},
# so pywb's Jinja2 templates find our warc_server/templates dir
'template_packages': ['warc_server', 'pywb'],
})
def test_invalid_config(self):
with raises(IOError):
init_app(create_wb_router,
load_yaml=True,
config_file='x-invalid-x')
def setup(self):
self.app = init_app(create_perms_checker_app,
load_yaml=True,
config_file=self.TEST_CONFIG)
self.testapp = webtest.TestApp(self.app)
def setup(self):
self.app = init_app(create_wb_router,
load_yaml=True,
config_file=self.TEST_CONFIG)
self.testapp = webtest.TestApp(self.app)
# this is a wsgi application that gets included with its own url prefix # alongside the main Django app in wsgi.py from pywb.framework.wsgi_wrappers import init_app from pywb_config import create_perma_pywb_app application = init_app(create_perma_pywb_app, load_yaml=False)
def _create_app(self):
self.app = init_app(create_wb_router)
self.testapp = webtest.TestApp(self.app)
from pywb.framework.wsgi_wrappers import init_app from pywb.webapp.pywb_init import create_wb_router #================================================================= # init pywb app #================================================================= application = init_app(create_wb_router, load_yaml=True)
def test_invalid_config(self):
with raises(IOError):
init_app(create_wb_router,
load_yaml=True,
config_file='x-invalid-x')
from pywb.framework.wsgi_wrappers import init_app, start_wsgi_server
# from pywb.core.cdx_api_handler import create_cdx_server_app
from pywb.webapp.pywb_init import create_cdx_server_app
# =================================================================
# init cdx server app
# =================================================================
application = init_app(create_cdx_server_app, load_yaml=True)
def main(): # pragma: no cover
start_wsgi_server(application, "CDX Server", default_port=8090)
if __name__ == "__main__":
main()
def load(self):
super(CdxCli, self).load()
return init_app(create_cdx_server_app,
load_yaml=True)
# this is a wsgi application that gets included with its own url prefix
# alongside the main Django app in wsgi.py
from django.conf import settings
from pywb.framework.wsgi_wrappers import init_app
from .pywb_config import (PermaCDXServer,
PermaHandler,
create_perma_wb_router,
get_archive_path)
application = init_app(create_perma_wb_router,
load_yaml=False,
config={
'port': 8000,
'collections': {'': 'PermaCDXSource'},
'archive_paths': str(get_archive_path(), 'utf-8'),
'server_cls': PermaCDXServer,
'wb_handler_class': PermaHandler,
'enable_memento': True,
'framed_replay': False,
# pywb template vars (used in templates called by pywb, such as head_insert.html, but not our ErrorTemplateView)
'template_globals': {
'static_path': settings.STATIC_URL.rstrip('/')+'/pywb'
},
# so pywb's Jinja2 templates find our warc_server/templates dir
'template_packages': ['warc_server', 'pywb'],
})
# this is a wsgi application that gets included with its own url prefix
# alongside the main Django app in wsgi.py
from pywb.framework.wsgi_wrappers import init_app
from pywb_config import create_perma_pywb_app
application = init_app(create_perma_pywb_app,
load_yaml=False)
PermaHandler,
create_perma_wb_router)
# monkey-patch WSGIApp.handle_exception to log exceptions as errors
real_handle_exception = WSGIApp.handle_exception
def handle_exception(self, env, exc, print_trace):
if print_trace:
try:
extra = {'request':WSGIRequest(env)}
except:
extra = {}
logging.error(traceback.format_exc(exc), extra=extra)
return real_handle_exception(self, env, exc, print_trace)
WSGIApp.handle_exception = handle_exception
# must be ascii, for some reason, else you'll get
# 'unicode' object has no attribute 'get'
path = default_storage.path('').encode('ascii', 'ignore') + '/'
application = init_app(create_perma_wb_router,
load_yaml=False,
config={
'port': 8000,
'collections': {'': 'PermaCDXSource'},
'archive_paths': path,
'server_cls': PermaCDXServer,
'wb_handler_class': PermaHandler,
'enable_memento': True
})
