def handle_cert_install(self, env):
if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'):
available = (self.ca is not None)
if self.proxy_cert_dl_view:
return (self.proxy_cert_dl_view.
render_response(available=available,
pem_path=self.CERT_DL_PEM,
p12_path=self.CERT_DL_P12))
elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM:
if not self.ca:
return None
buff = ''
with open(self.ca.ca_file, 'rb') as fh:
buff = fh.read()
content_type = 'application/x-x509-ca-cert'
return WbResponse.text_response(buff,
content_type=content_type)
elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12:
if not self.ca:
return None
buff = self.ca.get_root_PKCS12()
content_type = 'application/x-pkcs12'
return WbResponse.text_response(buff,
content_type=content_type)
def _get_video_info(self, wbrequest, info_url=None, video_url=None):
if not video_url:
video_url = wbrequest.wb_url.url
if not info_url:
info_url = wbrequest.wb_url.url
cache_key = None
if self.recording:
cache_key = self._get_cache_key('v:', video_url)
info = self.live_fetcher.get_video_info(video_url)
if info is None: #pragma: no cover
msg = ('youtube-dl is not installed, pip install youtube-dl to ' +
'enable improved video proxy')
return WbResponse.text_response(text=msg, status='404 Not Found')
#if info and info.formats and len(info.formats) == 1:
content_type = self.YT_DL_TYPE
metadata = json.dumps(info)
if (self.recording and cache_key):
headers = self._live_request_headers(wbrequest)
headers['Content-Type'] = content_type
if info_url.startswith('https://'):
info_url = info_url.replace('https', 'http', 1)
response = self.live_fetcher.add_metadata(info_url, headers, metadata)
self._cache[cache_key] = '1'
return WbResponse.text_response(metadata, content_type=content_type)
def _get_video_info(self, wbrequest, info_url=None, video_url=None):
if not video_url:
video_url = wbrequest.wb_url.url
if not info_url:
info_url = wbrequest.wb_url.url
cache_key = None
if self.recording:
cache_key = self._get_cache_key('v:', video_url)
info = self.live_fetcher.get_video_info(video_url)
if info is None: #pragma: no cover
msg = ('youtube-dl is not installed, pip install youtube-dl to ' +
'enable improved video proxy')
return WbResponse.text_response(text=msg, status='404 Not Found')
#if info and info.formats and len(info.formats) == 1:
content_type = self.YT_DL_TYPE
metadata = json.dumps(info)
if (self.recording and cache_key):
headers = self._live_request_headers(wbrequest)
headers['Content-Type'] = content_type
if info_url.startswith('https://'):
info_url = info_url.replace('https', 'http', 1)
response = self.live_fetcher.add_metadata(info_url, headers, metadata)
self._cache[cache_key] = '1'
return WbResponse.text_response(metadata, content_type=content_type)
def test_resp_1():
resp = vars(WbResponse.text_response('Test'))
expected = {'body': [b'Test'], 'status_headers': StatusAndHeaders(protocol = '', statusline = '200 OK',
headers = [('Content-Type', 'text/plain; charset=utf-8'), ('Content-Length', '4')])}
assert(resp == expected)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
if (status_headers.get_statuscode() == '500' and
status_headers.get_header('X-Archive-Orig-x-warcprox-error') == '500'):
msg = 'This url could not be recorded: '
raise LiveResourceException(msg, url=wbrequest.wb_url.url)
return WbResponse(status_headers, gen)
def __call__(self, wbrequest):
params = self.extract_params_from_wsgi_env(wbrequest.env)
cdx_iter = self.index_handler.load_cdx(wbrequest, params)
return WbResponse.text_stream(cdx_iter,
content_type='text/plain')
def render_search_page(self, wbrequest, **kwargs):
if self.search_view:
return self.search_view.render_response(wbrequest=wbrequest,
prefix=wbrequest.wb_prefix,
**kwargs)
else:
return WbResponse.text_response('No Lookup Url Specified')
def test_resp_2():
resp = vars(WbResponse.bin_stream([b'Test', b'Another'], content_type='text/plain; charset=utf-8', status='404'))
expected = {'body': [b'Test', b'Another'], 'status_headers': StatusAndHeaders(protocol = '', statusline = '404',
headers = [('Content-Type', 'text/plain; charset=utf-8')])}
assert(resp == expected)
def render_response(self, **kwargs):
template_result = self.render_to_string(**kwargs)
status = kwargs.get('status', '200 OK')
content_type = kwargs.get('content_type', 'text/html; charset=utf-8')
return WbResponse.text_response(template_result,
status=status,
content_type=content_type)
def render_response(self, **kwargs):
template_result = self.render_to_string(**dict(kwargs,
STATIC_URL=settings.STATIC_URL,
DEBUG=settings.DEBUG))
status = kwargs.get('status', '200 OK')
content_type = kwargs.get('content_type', 'text/html; charset=utf-8')
return WbResponse.text_response(template_result.encode('utf-8'), status=status, content_type=content_type)
def render_response(self, status='200 OK', content_type='text/html; charset=utf-8', **template_kwargs):
template_context = dict(
template_kwargs,
status=status,
content_type=content_type)
template_result = loader.render_to_string(self.filename, template_context, request=self.fake_request)
return WbResponse.text_response(unicode(template_result), status=status, content_type=content_type)
def __call__(self, wbrequest):
url = wbrequest.wb_url_str.split('?')[0]
full_path = self.static_path + url
try:
data = self.block_loader.load(full_path)
try:
data.seek(0, 2)
size = data.tell()
data.seek(0)
headers = [('Content-Length', str(size))]
except IOError:
headers = None
if 'wsgi.file_wrapper' in wbrequest.env:
reader = wbrequest.env['wsgi.file_wrapper'](data)
else:
reader = iter(lambda: data.read(), '')
content_type = 'application/octet-stream'
guessed = mimetypes.guess_type(full_path)
if guessed[0]:
content_type = guessed[0]
return WbResponse.text_stream(data,
content_type=content_type,
headers=headers)
except IOError:
raise NotFoundException('Static File Not Found: ' +
wbrequest.wb_url_str)
def __call__(self, wbrequest):
wb_url = wbrequest.wb_url
res = redis_client.get_all_embeds(wb_url)
return WbResponse.text_response(json.dumps(res),
content_type='application/json')
def render_response(self, **kwargs):
template_result = self.render_to_string(**kwargs)
status = kwargs.get('status', '200 OK')
content_type = kwargs.get('content_type', 'text/html; charset=utf-8')
return WbResponse.text_response(template_result.encode('utf-8'),
status=status,
content_type=content_type)
def __call__(self, wbrequest):
url = wbrequest.wb_url_str.split('?')[0]
full_path = self.static_path + url
try:
data = self.block_loader.load(full_path)
data.seek(0, 2)
size = data.tell()
data.seek(0)
headers = [('Content-Length', str(size))]
if 'wsgi.file_wrapper' in wbrequest.env:
reader = wbrequest.env['wsgi.file_wrapper'](data)
else:
reader = iter(lambda: data.read(), '')
content_type = 'application/octet-stream'
guessed = mimetypes.guess_type(full_path)
if guessed[0]:
content_type = guessed[0]
return WbResponse.text_stream(reader,
content_type=content_type,
headers=headers)
except IOError:
raise NotFoundException('Static File Not Found: ' +
wbrequest.wb_url_str)
def __call__(self, wbrequest):
"""
If someone requests a bare GUID url like /warc/1234-5678/, forward them to the submitted_url playback for that GUID.
"""
if wbrequest.wb_url_str == '/':
return WbResponse.redir_response("%s/%s/%s" % (settings.WARC_ROUTE, wbrequest.custom_params['guid'], wbrequest.custom_params['url']), status='301 Moved Permanently')
return super(PermaGUIDHandler, self).__call__(wbrequest)
def render_search_page(self, wbrequest, **kwargs):
if self.search_view:
return self.search_view.render_response(wbrequest=wbrequest,
prefix=wbrequest.wb_prefix,
**kwargs)
else:
return WbResponse.text_response('No Lookup Url Specified')
def make_cdx_response(self, wbrequest, cdx_iter, output, **kwargs):
# if not text, the iterator is assumed to be CDXObjects
if output and output != 'text':
view = self.views.get(output)
if view:
return view.render_response(wbrequest, cdx_iter, **kwargs)
return WbResponse.text_stream(cdx_iter)
def test_resp_3():
resp = vars(WbResponse.redir_response('http://example.com/otherfile'))
expected = {'body': [], 'status_headers': StatusAndHeaders(protocol = '', statusline = '302 Redirect',
headers = [('Location', 'http://example.com/otherfile'), ('Content-Length', '0')])}
assert(resp == expected)
def make_cdx_response(self, wbrequest, cdx_iter, output, **kwargs):
# if not text, the iterator is assumed to be CDXObjects
if output and output != 'text':
view = self.views.get(output)
if view:
return view.render_response(wbrequest, cdx_iter, **kwargs)
return WbResponse.text_stream(cdx_iter)
def handle_exception(self, env, exc, print_trace):
error_view = None
if hasattr(self.wb_router, 'error_view'):
error_view = self.wb_router.error_view
if hasattr(exc, 'status'):
if callable(exc.status):
status = exc.status()
else:
status = exc.status
# wsgi requires status
# - to have at least 4 characters and
# - to start with a number / integer
if type(status) == int:
status = '{} Exception {}'.format(status, type(exc).__name__)
elif type(status) == str and status[0].isdigit():
pass
else:
status = '500 Internal Server Error'
else:
status = '500 Internal Server Error'
if hasattr(exc, 'url'):
err_url = exc.url
else:
err_url = None
if len(exc.args):
err_msg = str(exc.args[0])
if print_trace:
import traceback
err_details = traceback.format_exc()
print(err_details)
else:
logging.info(err_msg)
err_details = None
if error_view:
if err_url and isinstance(err_url, str):
err_url = to_native_str(err_url, 'utf-8')
if err_msg and isinstance(err_msg, str):
err_msg = to_native_str(err_msg, 'utf-8')
return error_view.render_response(exc_type=type(exc).__name__,
err_msg=err_msg,
err_details=err_details,
status=status,
env=env,
err_url=err_url)
else:
msg = status + ' Error: '
if err_msg:
msg += err_msg
#msg = msg.encode('utf-8', 'ignore')
return WbResponse.text_response(msg, status=status)
def __call__(self, wbrequest):
params = self.extract_params_from_wsgi_env(wbrequest.env)
try:
cdx_iter = self.index_handler.load_cdx(wbrequest, params)
except NotFoundException:
msg = 'No Captures found for: ' + params.get('url')
if params.get('output') == 'json':
msg = json.dumps(dict(error=msg))
content_type='application/json'
else:
content_type='text/plain'
return WbResponse.text_response(msg, content_type=content_type,
status='404 Not Found')
return WbResponse.text_stream(cdx_iter,
content_type='text/plain')
def make_redir_response(self, url, headers=None):
if not headers:
headers = []
if self.extra_headers:
for name, value in six.iteritems(self.extra_headers):
headers.append((name, value))
return WbResponse.redir_response(url, headers=headers)
def __call__(self, wbrequest):
params = self.extract_params_from_wsgi_env(wbrequest.env)
try:
cdx_iter = self.index_handler.load_cdx(wbrequest, params)
except NotFoundException:
msg = 'No Captures found for: ' + params.get('url')
if params.get('output') == 'json':
msg = json.dumps(dict(error=msg))
content_type = 'application/json'
else:
content_type = 'text/plain'
return WbResponse.text_response(msg,
content_type=content_type,
status='404 Not Found')
return WbResponse.text_stream(cdx_iter, content_type='text/plain')
def make_redir_response(self, url, headers=None):
if not headers:
headers = []
if self.extra_headers:
for name, value in six.iteritems(self.extra_headers):
headers.append((name, value))
return WbResponse.redir_response(url, headers=headers)
def __call__(self, wbrequest):
"""
If someone requests a bare GUID url like /warc/1234-5678/, forward them to the submitted_url playback for that GUID.
"""
if wbrequest.wb_url_str == '/':
return WbResponse.redir_response("/warc/%s/%s" %
(wbrequest.custom_params['guid'],
wbrequest.custom_params['url']),
status='301 Moved Permanently')
return super(PermaGUIDHandler, self).__call__(wbrequest)
def render_response(self, wbrequest, cdx_lines, **kwargs):
memento_lines = make_timemap(wbrequest, cdx_lines)
new_memento_lines = self.fix_timegate_line(memento_lines)
response = WbResponse.text_stream(new_memento_lines, content_type=LINK_FORMAT, )
response.status_headers.headers.append(('Cache-Control',
'max-age={}'.format(settings.CACHE_MAX_AGES['timemap'])))
return response
def test_resp_4():
resp = vars(WbResponse.text_response('Test').add_range(10, 4, 100))
expected = {'body': [b'Test'], 'status_headers': StatusAndHeaders(protocol = '', statusline = '206 Partial Content',
headers = [ ('Content-Type', 'text/plain; charset=utf-8'),
('Content-Length', '4'),
('Content-Range', 'bytes 10-13/100'),
('Accept-Ranges', 'bytes')])}
assert(resp == expected)
def _get_video_info(self, wbrequest, info_url=None, video_url=None):
if not self.youtubedl:
self.youtubedl = YoutubeDLWrapper()
if not video_url:
video_url = wbrequest.wb_url.url
if not info_url:
info_url = wbrequest.wb_url.url
cache_key = None
if self.proxies:
cache_key = self._get_cache_key('v:', video_url)
info = self.youtubedl.extract_info(video_url)
if info is None: #pragma: no cover
msg = ('youtube-dl is not installed, pip install youtube-dl to ' +
'enable improved video proxy')
return WbResponse.text_response(text=msg, status='404 Not Found')
#if info and info.formats and len(info.formats) == 1:
content_type = self.YT_DL_TYPE
metadata = json.dumps(info)
if (self.proxies and cache_key):
headers = self._live_request_headers(wbrequest)
headers['Content-Type'] = content_type
info_url = HttpsUrlRewriter.remove_https(info_url)
response = requests.request(method='PUTMETA',
url=info_url,
data=metadata,
headers=headers,
proxies=self.proxies,
verify=False)
self._cache[cache_key] = '1'
return WbResponse.text_response(metadata, content_type=content_type)
def select_coll_response(self, env, default_coll=None):
proxy_msg = 'Basic realm="{0}"'.format(self.auth_msg)
headers = [('Content-Type', 'text/plain'),
('Proxy-Authenticate', proxy_msg)]
status_headers = StatusAndHeaders('407 Proxy Authentication', headers)
value = self.auth_msg
return WbResponse(status_headers, value=[value.encode('utf-8')])
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# if cookie set, pass recorded timestamp info via cookie
# so that client side may be able to access it
# used by framed mode to update frame banner
if self.live_cookie:
cdx = wbrequest.env.get('pywb.cdx')
if cdx:
value = self.live_cookie.format(cdx['timestamp'])
status_headers.headers.append(('Set-Cookie', value))
return WbResponse(status_headers, gen)
def check_single_url(self, wbrequest, perms_checker):
urlkey = self.url_canon(wbrequest.wb_url.url)
if not perms_checker.allow_url_lookup(urlkey):
response_text = BLOCK
else:
response_text = ALLOW
#TODO: other types of checking
return WbResponse.text_response(response_text,
content_type=RESPONSE_TYPE)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header("Content-Type")
tpl_name = self.templates.get(content_type)
if tpl_name is not None:
tpl = env.get_template(tpl_name)
result = tpl.render(url=wbrequest.wb_url.url)
return WbResponse.text_response(result.encode("utf-8-sig"), content_type="text/html")
return super(TemplateRewriteHandler, self)._make_response(wbrequest, status_headers, gen, is_rewritten)
def check_single_url(self, wbrequest, perms_checker):
urlkey = self.url_canon(wbrequest.wb_url.url)
urlkey = urlkey.encode('utf-8')
if not perms_checker.allow_url_lookup(urlkey):
response_text = BLOCK
else:
response_text = ALLOW
#TODO: other types of checking
return WbResponse.text_response(response_text,
content_type=RESPONSE_TYPE)
def render_response(self,
status='200 OK',
content_type='text/html; charset=utf-8',
**template_kwargs):
template_context = dict(template_kwargs,
status=status,
content_type=content_type)
template_result = loader.render_to_string(self.filename,
template_context,
request=self.fake_request)
return WbResponse.text_response(unicode(template_result),
status=status,
content_type=content_type)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header('Content-Type')
redir = self.redirects.get(content_type)
if redir:
return WbResponse.redir_response(redir.format(wbrequest.wb_url.url))
return super(CustomRedirHandler, self)._make_response(wbrequest,
status_headers,
gen,
is_rewritten)
def render_response(self, wbrequest, cdx_lines, **kwargs):
memento_lines = make_timemap(wbrequest, cdx_lines)
new_memento_lines = self.fix_timegate_line(memento_lines)
response = WbResponse.text_stream(
new_memento_lines,
content_type=LINK_FORMAT,
)
response.status_headers.headers.append(
('Cache-Control',
'max-age={}'.format(settings.CACHE_MAX_AGES['timemap'])))
return response
def render_response(self,
status='200 OK',
content_type='text/html; charset=utf-8',
**template_kwargs):
template_context = dict(template_kwargs,
status=status,
content_type=content_type)
template_result = loader.render_to_string(self.filename,
template_context,
request=self.fake_request)
# We have to cast the Django SafeText class to str because wsgiref can't handle subclasses
return WbResponse.text_response(str(template_result),
status=status,
content_type=content_type)
def test_resp_1():
resp = vars(WbResponse.text_response('Test'))
expected = {
'body': [b'Test'],
'status_headers':
StatusAndHeaders(protocol='',
statusline='200 OK',
headers=[('Content-Type',
'text/plain; charset=utf-8'),
('Content-Length', '4')])
}
assert (resp == expected)
def test_resp_3():
resp = vars(WbResponse.redir_response('http://example.com/otherfile'))
expected = {
'body': [],
'status_headers':
StatusAndHeaders(protocol='',
statusline='302 Redirect',
headers=[('Location', 'http://example.com/otherfile'),
('Content-Length', '0')])
}
assert (resp == expected)
def handle_cert_install(self, env):
if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'):
available = (self.ca is not None)
if self.proxy_cert_dl_view:
return (self.proxy_cert_dl_view.render_response(
available=available,
pem_path=self.CERT_DL_PEM,
p12_path=self.CERT_DL_P12))
elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM:
if not self.ca:
return None
buff = b''
with open(self.ca.ca_file, 'rb') as fh:
buff = fh.read()
content_type = 'application/x-x509-ca-cert'
headers = [('Content-Length', str(len(buff)))]
return WbResponse.bin_stream([buff],
content_type=content_type,
headers=headers)
elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12:
if not self.ca:
return None
buff = self.ca.get_root_PKCS12()
content_type = 'application/x-pkcs12'
headers = [('Content-Length', str(len(buff)))]
return WbResponse.bin_stream([buff],
content_type=content_type,
headers=headers)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header('Content-Type')
tpl_name = self.templates.get(content_type)
if tpl_name is not None:
tpl = env.get_template(tpl_name)
result = tpl.render(url=wbrequest.wb_url.url)
return WbResponse.text_response(result.encode('utf-8-sig'),
content_type='text/html')
return super(TemplateRewriteHandler,
self)._make_response(wbrequest, status_headers, gen,
is_rewritten)
def test_resp_4():
resp = vars(WbResponse.text_response('Test').add_range(10, 4, 100))
expected = {
'body': [b'Test'],
'status_headers':
StatusAndHeaders(protocol='',
statusline='206 Partial Content',
headers=[('Content-Type',
'text/plain; charset=utf-8'),
('Content-Length', '4'),
('Content-Range', 'bytes 10-13/100'),
('Accept-Ranges', 'bytes')])
}
assert (resp == expected)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header('Content-Type')
redir = self.redirects.get(content_type)
# safe is used so output matches JavaScript's encodeURIComponent()
url = urllib.quote(unicode(wbrequest.wb_url.url).encode('utf-8'),
safe='~()*!.\'')
if redir:
return WbResponse.redir_response(redir.format(url))
return super(CustomRedirHandler,
self)._make_response(wbrequest, status_headers, gen,
is_rewritten)
def test_resp_2():
resp = vars(
WbResponse.bin_stream([b'Test', b'Another'],
content_type='text/plain; charset=utf-8',
status='404'))
expected = {
'body': [b'Test', b'Another'],
'status_headers':
StatusAndHeaders(protocol='',
statusline='404',
headers=[('Content-Type', 'text/plain; charset=utf-8')
])
}
assert (resp == expected)
def handle_exception(self, env, exc, print_trace):
error_view = None
if hasattr(self.wb_router, 'error_view'):
error_view = self.wb_router.error_view
if hasattr(exc, 'status'):
status = exc.status()
else:
status = '500 Internal Server Error'
if hasattr(exc, 'url'):
err_url = exc.url
else:
err_url = None
if len(exc.args):
err_msg = exc.args[0]
if print_trace:
import traceback
err_details = traceback.format_exc()
print(err_details)
else:
logging.info(err_msg)
err_details = None
if error_view:
if err_url and isinstance(err_url, str):
err_url = to_native_str(err_url, 'utf-8')
if err_msg and isinstance(err_msg, str):
err_msg = to_native_str(err_msg, 'utf-8')
return error_view.render_response(exc_type=type(exc).__name__,
err_msg=err_msg,
err_details=err_details,
status=status,
env=env,
err_url=err_url)
else:
msg = status + ' Error: '
if err_msg:
msg += err_msg
#msg = msg.encode('utf-8', 'ignore')
return WbResponse.text_response(msg,
status=status)
def handle_exception(self, env, exc, print_trace):
error_view = None
if hasattr(self.wb_router, 'error_view'):
error_view = self.wb_router.error_view
if hasattr(exc, 'status'):
status = exc.status()
else:
status = '500 Internal Server Error'
if hasattr(exc, 'url'):
err_url = exc.url
else:
err_url = None
if len(exc.args):
err_msg = exc.args[0]
if print_trace:
import traceback
err_details = traceback.format_exc()
print(err_details)
else:
logging.info(err_msg)
err_details = None
if error_view:
if err_url and isinstance(err_url, str):
err_url = to_native_str(err_url, 'utf-8')
if err_msg and isinstance(err_msg, str):
err_msg = to_native_str(err_msg, 'utf-8')
return error_view.render_response(exc_type=type(exc).__name__,
err_msg=err_msg,
err_details=err_details,
status=status,
env=env,
err_url=err_url)
else:
msg = status + ' Error: '
if err_msg:
msg += err_msg
#msg = msg.encode('utf-8', 'ignore')
return WbResponse.text_response(msg,
status=status)
def handle_request(self, wbrequest):
if wbrequest.wb_url.is_query():
type_ = wbrequest.wb_url.LATEST_REPLAY
url = wbrequest.urlrewriter.get_new_url(type=type_, timestamp='')
return WbResponse.redir_response(url)
try:
return self.render_content(wbrequest)
except Exception as exc:
import traceback
err_details = traceback.format_exc(exc)
print err_details
url = wbrequest.wb_url.url
msg = 'Could not load the url from the live web: ' + url
raise LiveResourceException(msg=msg, url=url)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header('Content-Type')
redir = self.redirects.get(content_type)
# safe is used so output matches JavaScript's encodeURIComponent()
url = urllib.quote(unicode(wbrequest.wb_url.url).encode('utf-8'),
safe='~()*!.\'')
if redir:
return WbResponse.redir_response(redir.format(url))
return super(CustomRedirHandler, self)._make_response(wbrequest,
status_headers,
gen,
is_rewritten)
def handle_request(self, wbrequest):
if wbrequest.wb_url.is_query():
type_ = wbrequest.wb_url.LATEST_REPLAY
url = wbrequest.urlrewriter.get_new_url(type=type_, timestamp='')
return WbResponse.redir_response(url)
try:
return self.render_content(wbrequest)
except Exception as exc:
import traceback
err_details = traceback.format_exc()
print(err_details)
url = wbrequest.wb_url.url
msg = 'Could not load the url from the live web: ' + url
raise LiveResourceException(msg=msg, url=url)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header('Content-Type')
tpl_name = self.templates.get(content_type)
if tpl_name is not None:
tpl = env.get_template(tpl_name)
tpl_params = {'url': wbrequest.wb_url.url}
tpl_params.update(wbrequest.env.get('pywb.template_params',
{}))
result = tpl.render(**tpl_params)
return WbResponse.text_response(
result.encode('utf-8-sig'),
content_type=b'text/html; charset=utf-8')
return super(TemplateRewriteHandler,
self)._make_response(wbrequest, status_headers, gen,
is_rewritten)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header('Content-Type')
tpl_name = self.templates.get(content_type)
if tpl_name is not None:
tpl = env.get_template(tpl_name)
tpl_params = {'url': wbrequest.wb_url.url}
tpl_params.update(wbrequest.env.get('pywb.template_params', {}))
result = tpl.render(**tpl_params)
return WbResponse.text_response(result.encode('utf-8-sig'),
content_type=b'text/html; charset=utf-8')
return super(TemplateRewriteHandler, self)._make_response(
wbrequest,
status_headers,
gen,
is_rewritten)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header("Content-Type")
cleaned_content_type = _lookup_key(content_type)
tpl_name = self.templates.get(cleaned_content_type)
if tpl_name is not None:
tpl = env.get_template(tpl_name)
tpl_params = {"url": wbrequest.wb_url.url}
tpl_params.update(wbrequest.env.get("pywb.template_params", {}))
result = tpl.render(**tpl_params)
return WbResponse.text_response(
result.encode("utf-8-sig"), content_type=b"text/html; charset=utf-8"
)
return super(TemplateRewriteHandler, self)._make_response(
wbrequest, status_headers, gen, is_rewritten
)
def _make_response(self, wbrequest, status_headers, gen, is_rewritten):
# only redirect for non-identity and non-embeds
if not wbrequest.wb_url.is_embed and not wbrequest.wb_url.is_identity:
content_type = status_headers.get_header("Content-Type")
cleaned_content_type = _lookup_key(content_type)
tpl_name = self.templates.get(cleaned_content_type)
if tpl_name is not None:
tpl = env.get_template(tpl_name)
tpl_params = {"url": wbrequest.wb_url.url}
tpl_params.update(wbrequest.env.get("pywb.template_params",
{}))
result = tpl.render(**tpl_params)
return WbResponse.text_response(
result.encode("utf-8-sig"),
content_type=b"text/html; charset=utf-8")
return super(TemplateRewriteHandler,
self)._make_response(wbrequest, status_headers, gen,
is_rewritten)
def _get_video_info(self, wbrequest, info_url=None, video_url=None):
if not self.youtubedl:
self.youtubedl = YoutubeDLWrapper()
if not video_url:
video_url = wbrequest.wb_url.url
if not info_url:
info_url = wbrequest.wb_url.url
cache_key = None
if self.proxies:
cache_key = self._get_cache_key('v:', video_url)
info = self.youtubedl.extract_info(video_url)
#if info and info.formats and len(info.formats) == 1:
content_type = self.YT_DL_TYPE
metadata = json.dumps(info)
if (self.proxies and cache_key):
headers = self._live_request_headers(wbrequest)
headers['Content-Type'] = content_type
info_url = HttpsUrlRewriter.remove_https(info_url)
response = requests.request(method='PUTMETA',
url=info_url,
data=metadata,
headers=headers,
proxies=self.proxies,
verify=False)
self._cache[cache_key] = '1'
return WbResponse.text_response(metadata, content_type=content_type)
def __call__(self, wbrequest):
params = self.extract_params_from_wsgi_env(wbrequest.env)
cdx_iter = self.index_handler.load_cdx(wbrequest, params)
return WbResponse.text_stream(cdx_iter)
def render_response(self, wbrequest, cdx_lines, **kwargs):
memento_lines = make_timemap(wbrequest, cdx_lines)
return WbResponse.text_stream(memento_lines, content_type=LINK_FORMAT)
def render_response(self, **kwargs):
template_result = self.render_to_string(**kwargs)
status = kwargs.get("status", "200 OK")
content_type = kwargs.get("content_type", "text/html; charset=utf-8")
return WbResponse.text_response(template_result.encode("utf-8"), status=status, content_type=content_type)
def __call__(self, wbrequest):
return WbResponse.redir_response(self.redir_path +
wbrequest.wb_url_str)
def handle_connect(self, env):
sock = self.get_request_socket(env)
if not sock:
return WbResponse.text_response('HTTPS Proxy Not Supported',
'405 HTTPS Proxy Not Supported')
sock.send('HTTP/1.0 200 Connection Established\r\n')
sock.send('Server: pywb proxy\r\n')
sock.send('\r\n')
hostname, port = env['REL_REQUEST_URI'].split(':')
if not self.use_wildcard:
certfile = self.ca.cert_for_host(hostname)
else:
certfile = self.ca.get_wildcard_cert(hostname)
try:
ssl_sock = ssl.wrap_socket(sock,
server_side=True,
certfile=certfile,
#ciphers="ALL",
suppress_ragged_eofs=False,
ssl_version=ssl.PROTOCOL_SSLv23
)
env['pywb.proxy_ssl_sock'] = ssl_sock
buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE)
statusline = buffreader.readline().rstrip()
except Exception as se:
raise BadRequestException(se.message)
statusparts = statusline.split(' ')
if len(statusparts) < 3:
raise BadRequestException('Invalid Proxy Request: ' + statusline)
env['REQUEST_METHOD'] = statusparts[0]
env['REL_REQUEST_URI'] = ('https://' +
env['REL_REQUEST_URI'].replace(':443', '') +
statusparts[1])
env['SERVER_PROTOCOL'] = statusparts[2].strip()
env['pywb.proxy_scheme'] = 'https'
env['pywb.proxy_host'] = hostname
env['pywb.proxy_port'] = port
env['pywb.proxy_req_uri'] = statusparts[1]
queryparts = env['REL_REQUEST_URI'].split('?', 1)
env['PATH_INFO'] = queryparts[0]
env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else ''
env['pywb.proxy_query'] = env['QUERY_STRING']
while True:
line = buffreader.readline()
if line:
line = line.rstrip()
if not line:
break
parts = line.split(':', 1)
if len(parts) < 2:
continue
name = parts[0].strip()
value = parts[1].strip()
name = name.replace('-', '_').upper()
if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'):
name = 'HTTP_' + name
env[name] = value
remain = buffreader.rem_length()
if remain > 0:
remainder = buffreader.read(self.BLOCK_SIZE)
env['wsgi.input'] = BufferedReader(ssl_sock,
block_size=self.BLOCK_SIZE,
starting_data=remainder)