def wiki_path(self, slug):
url = URL(
scheme='https',
host='en.wikipedia.org',
path=['wiki', str(slug).replace(' ', '_')],
)
return url.to_text()
def urls(draw: Callable, collection: Optional[bool] = None) -> URL:
"""
Strategy that generates URLs.
"""
urlPathText = partial(text,
alphabet=characters(blacklist_characters="/?#"),
max_size=32)
segments = draw(lists(urlPathText(), max_size=16))
url = URL(
scheme=draw(sampled_from(("http", "https"))),
# FIXME: wimpy host name alphabet
host=draw(text(alphabet=ascii_letters, min_size=1)),
port=draw(integers(min_value=1, max_value=65535)),
path=segments,
)
if collection:
url = url.child("")
else:
if collection is not None:
url = url.child(draw(urlPathText(min_size=1)))
return url
def test_rooted_empty(self):
"""
Rooted empty path.
"""
self.assertThat([
url_path(URL.from_text(u'http://example.com/')),
url_path(URL.from_text(u'/'))
], AllMatch(Equals(u'/')))
def test_rooted(self):
"""
Rooted path.
"""
self.assertThat([
url_path(URL.from_text(u'http://example.com/foo/bar')),
url_path(URL.from_text(u'/foo/bar'))
], AllMatch(Equals(u'/foo/bar')))
def test_known_HTTP_methods_use_associated_renderer(self):
render = traversal.method_delegate(
GET=lambda request: b"foo",
put=lambda request: request.method,
)
get = Request(url=URL(path=[u""]), method=b"GET")
self.assertEqual(render(get), b"foo")
put = Request(url=URL(path=[u""]), method=b"PUT")
self.assertEqual(render(put), b"PUT")
def test_it_maps_multiple_routes_for_specified_methods(self):
self.mapper.add(b"/", lambda r: Response(b"GET"), methods=[b"GET"])
self.mapper.add(b"/", lambda r: Response(b"POST"), methods=[b"POST"])
get = Request(url=URL(path=[u""], rooted=True), method=b"GET")
post = Request(url=URL(path=[u""], rooted=True), method=b"POST")
map = self.mapper.map
self.assertEqual(
(map(get, path=b"/")(get), map(post, path=b"/")(post)),
(Response(b"GET"), Response(b"POST")),
)
def render(self, twistedRequest):
path = twistedRequest.uri.lstrip("/").decode("ascii").split(u"/")
request = Request(
content=twistedRequest.content,
headers=Headers(twistedRequest.requestHeaders.getAllRawHeaders()),
method=twistedRequest.method,
url=URL(
scheme=u"https" if twistedRequest.isSecure() else u"http",
host=twistedRequest.getRequestHostname().decode("ascii"),
path=path,
query=[(k.decode("ascii"), each.decode("ascii"))
for k, v in twistedRequest.args.items() for each in v],
),
)
response = self.application.serve(
request=request,
path=b"/" + b"/".join(twistedRequest.postpath),
)
twistedRequest.setResponseCode(response.code)
for k, v in response.headers.canonicalized():
twistedRequest.responseHeaders.setRawHeaders(k, v)
return response.content
def _nevow_request_to_request_map(req):
"""
Convert a Nevow request object into an immutable request map.
"""
headers = req.requestHeaders
content_type, character_encoding = _get_content_type(headers)
iri = URL.from_text(req.uri.decode('utf-8')).to_iri()
host = _get_first_header(headers, b'host').decode('utf-8')
scheme = u'https' if req.isSecure() else u'http'
if u':' in host:
host, port = host.split(u':', 1)
port = int(port)
else:
port = {u'https': 443, u'http': 80}.get(scheme)
return m(
body=req.content,
content_type=content_type,
content_length=_get_first_header(headers, b'content-length'),
character_encoding=character_encoding,
headers=freeze(dict(headers.getAllRawHeaders())),
remote_addr=req.getClientIP(),
request_method=req.method,
server_name=host,
server_port=port,
scheme=scheme,
#ssl_client_cert=XXX,
uri=iri,
#query_string
path_info=url_path(iri),
protocol=getattr(req, 'clientproto', None))
def retarget_links(html_tree, mode='external'):
if mode == 'none':
return
elif mode not in ('external', 'all'):
raise ValueError('expected "none", "external", or "all", not: %r'
% mode)
for el in html_tree.iter():
if not isinstance(el.tag, basestring):
continue
if not el.tag == 'a':
continue
if el.get('target'):
continue # let explicit settings lie
href = el.get('href')
if not href or href.startswith('#'):
continue
if mode == 'all':
retarget = True
elif mode == 'external':
try:
url = URL.from_text(href)
except ValueError:
retarget = True
else:
retarget = bool(url.host)
if retarget:
el.set('target', '_blank')
el.set('rel', 'noopener')
return
def _a_handler(a_el):
href = a_el.attrib.get('href')
if href:
url = urljoin(a_el.base_url, href)
if not isinstance(url, six.text_type):
url = six.u(url)
return URL.from_text(url)
def setUp(self):
service = StubFeed({
URL.from_text(feed._source).host.encode('ascii'): makeXML(feed)
for feed in FEEDS
})
treq = StubTreq(service.resource())
self.retriever = FeedRetrieval(treq=treq)
def _thumbnail_url(self, embed_url) -> DecodedURL:
"""
Generate the URL of the thumbnail for a YouTube video embed.
There are a few apparent options for the filename in this URL:
default.jpg — always seems to be available, even on old videos, but is tiny (120×90).
mqdefault.jpg — always seems to be available, and has a more contemporary
aspect ratio (320×180). This is what this function returns.
hqdefault.jpg — always seems to be available (though I haven't tested
videos more than 5 years old), but it is always 4:3 (480×360), so recent
videos are letterboxed.
maxresdefault.jpg — only recent videos seem to have this (e.g., all of
the videos I've checked on the YouTube homepage as of October 2017 do).
An old video which lacks maxresdefault.jpg: https://www.youtube.com/watch?v=XPIFncE22pw
A recent video from the homepage which has all of them: https://www.youtube.com/watch?v=R1ZXOOLMJ8s
See also https://boingboing.net/features/getthumbs.html
"""
video_id = embed_url.path[1]
return DecodedURL(
URL(
scheme="https",
host="i.ytimg.com",
path=("vi", video_id, "mqdefault.jpg"),
), )
def render_GET(self, req):
"""
Historically, accessing this via "GET /uri?uri=<capabilitiy>"
was/is a feature -- which simply redirects to the more-common
"GET /uri/<capability>" with any other query args
preserved. New code should use "/uri/<cap>"
"""
uri_arg = req.args.get(b"uri", [None])[0]
if uri_arg is None:
raise WebError("GET /uri requires uri=")
# shennanigans like putting "%2F" or just "/" itself, or ../
# etc in the <cap> might be a vector for weirdness so we
# validate that this is a valid capability before proceeding.
cap = uri.from_string(uri_arg)
if isinstance(cap, uri.UnknownURI):
raise WebError("Invalid capability")
# so, using URL.from_text(req.uri) isn't going to work because
# it seems Nevow was creating absolute URLs including
# host/port whereas req.uri is absolute (but lacks host/port)
redir_uri = URL.from_text(req.prePathURL().decode('utf8'))
redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8'))
# add back all the query args that AREN'T "?uri="
for k, values in req.args.items():
if k != b"uri":
for v in values:
redir_uri = redir_uri.add(k.decode('utf8'), v.decode('utf8'))
return redirectTo(redir_uri.to_text().encode('utf8'), req)
def test_basic(self):
"""
Test basic request map keys.
"""
request = fake_twisted_request(request_headers={
b'x-foo': [b'bar'],
})
self.assertThat(
_nevow_request_to_request_map(request),
ContainsDict({
'content_type':
Equals(b'application/octet-stream'),
'content_length':
Equals(0),
'character_encoding':
Is(None),
'headers':
Equals({
b'Content-Length': [0],
b'X-Foo': [b'bar'],
b'Host': [b'example.com']
}),
'remote_addr':
Equals(b'192.168.1.1'),
'request_method':
Equals(b'GET'),
'server_name':
Equals(b'example.com'),
'server_port':
Equals(80),
'scheme':
Equals(b'http'),
'uri':
Equals(URL.from_text(u'/one'))
}))
def construct(
cls,
host=None,
user=None,
path=None,
):
# test whether the path is rooted
rooted = path.startswith('/')
# the path must be rooted when a remote host is given
if host is not None and not rooted:
raise ValueError(
"Path must be a fully rooted path when a host is given")
# get the sections of the path
path_segs = path.strip('/').split('/')
if user is None:
user = ''
url = URL(
scheme='rsync',
host=host,
userinfo=user,
rooted=rooted,
path=path_segs,
)
return Endpoint(url)
def uri(self):
# type: () -> URL
request = self._request
# This code borrows from t.w.server.Request._prePathURL.
if request.isSecure():
scheme = u"https"
else:
scheme = u"http"
netloc = nativeString(request.getRequestHostname())
port = request.getHost().port
if request.isSecure():
default = 443
else:
default = 80
if port != default:
netloc += u":{}".format(port)
path = nativeString(request.uri)
if path and path[0] == u"/":
path = path[1:]
return URL.fromText(u"{}://{}/{}".format(scheme, netloc, path))
def retarget_links(html_tree, mode='external'):
if mode == 'none':
return
elif mode not in ('external', 'all'):
raise ValueError('expected "none", "external", or "all", not: %r' %
mode)
for el in html_tree.iter():
if not isinstance(el.tag, basestring):
continue
if not el.tag == 'a':
continue
if el.get('target'):
continue # let explicit settings lie
href = el.get('href')
if not href or href.startswith('#'):
continue
if mode == 'all':
retarget = True
elif mode == 'external':
try:
url = URL.from_text(href)
except ValueError:
retarget = True
else:
retarget = bool(url.host)
if retarget:
el.set('target', '_blank')
el.set('rel', 'noopener')
return
class TestUnicodeRenderer(TestCase):
request = Request(url=URL(path=[u""]))
def test_it_renders_via_the_given_encoding(self):
renderer = renderers.Unicode(encoding="utf-8")
render = renderers.bind(renderer, to=lambda _: u"שלום")
self.assertEqual(
render(self.request), Response(u"שלום".encode("utf-8")),
)
def test_encoding_ignoring_errors(self):
renderer = renderers.Unicode(encoding="ascii", errors="ignore")
render = renderers.bind(renderer, to=lambda _: u"שלום")
self.assertEqual(render(self.request), Response(b""))
def test_encoding_errors_by_default(self):
renderer = renderers.Unicode(encoding="ascii")
render = renderers.bind(renderer, to=lambda _: u"שלום")
with self.assertRaises(UnicodeEncodeError):
render(self.request)
def test_UTF8(self):
render = renderers.bind(renderers.UTF8, to=lambda _: u"שלום")
self.assertEqual(
render(self.request), Response(u"שלום".encode("utf-8")),
)
def requestRedirectError(app: Any, request: IRequest,
failure: Failure) -> KleinRenderable:
"""
Redirect.
"""
url = URL.fromText(failure.value.args[0])
return redirect(request, url)
def fake_nevow_request(method='GET',
body=b'',
is_secure=False,
uri=b'http://example.com/one',
request_headers=None,
Request=NevowRequest):
"""
Create a fake `NevowRequest` instance for the purposes of testing.
"""
uri = URL.from_text(uri.decode('utf-8')).to_uri()
channel = DummyChannel()
if is_secure:
channel.transport = DummyChannel.SSL()
request = Request(channel=channel)
request.method = method
request.uri = url_path(uri)
request.clientproto = b'HTTP/1.1'
request.client = channel.transport.getPeer()
content_length = len(body)
request.requestHeaders.setRawHeaders('host',
[uri.authority().encode('utf-8')])
request.requestHeaders.setRawHeaders('content-length', [content_length])
if request_headers:
for k, v in request_headers.items():
request.requestHeaders.setRawHeaders(k, v)
request.gotLength(content_length)
request.content.write(body)
request.content.seek(0)
return request
async def cacheFromURL(self, url: URL, name: str) -> Path:
"""
Download a resource and cache it.
"""
cacheDir = self.config.CachedResourcesPath
cacheDir.mkdir(exist_ok=True)
destination = cacheDir / name
if not destination.exists():
with NamedTemporaryFile(dir=str(cacheDir),
delete=False,
suffix=".tmp") as tmp:
path = Path(tmp.name)
try:
await downloadPage(url.asText().encode("utf-8"), tmp)
except BaseException as e:
self._log.failure("Download failed for {url}: {error}",
url=url,
error=e)
try:
path.unlink()
except (OSError, IOError) as e:
self._log.critical(
"Failed to remove temporary file {path}: {error}",
path=path,
error=e)
else:
path.rename(destination)
return destination
def test_specified_renderer_multiple_routes(self):
self.router.add(b"/", view, methods=[b"GET"], renderer=None)
self.router.add(
b"/",
view,
methods=[b"POST"],
renderer=ReverseRenderer(),
)
get = Request(url=URL(path=[u""]), method=b"GET")
response = self.router.route(get, path=b"/")
self.assertEqual(response, Response(b"{}"))
post = Request(url=URL(path=[u""]), method=b"POST")
response = self.router.route(post, path=b"/")
self.assertEqual(response, Response(b"}{"))
def uri(self):
# type: () -> URL
request = self._request
# This code borrows from t.w.server.Request._prePathURL.
if request.isSecure():
scheme = u"https"
else:
scheme = u"http"
netloc = nativeString(request.getRequestHostname())
port = request.getHost().port
if request.isSecure():
default = 443
else:
default = 80
if port != default:
netloc += u":{}".format(port)
path = nativeString(request.uri)
if path and path[0] == u"/":
path = path[1:]
return URL.fromText(u"{}://{}/{}".format(scheme, netloc, path))
def test_it_maps_routes_for_specified_methods(self):
self.mapper.add(b"/route", view, methods=[b"POST"])
request = Request(
url=URL(path=[u"route"], rooted=True),
method=b"POST",
)
self.assertEqual(self.mapper.map(request, path=b"/route"), view)
def get_pull_requests(url: hyperlink.URL) -> t.List[str]:
url = url.replace(host="api.github.com",
path=("repos", ) + url.path + ("pulls", ))
r = requests.get(url)
r.raise_for_status()
pulls = r.json()
out = []
assert isinstance(pulls, list)
for pull in pulls:
head = pull["head"]
if head is None:
continue
repo = head["repo"]
if repo is None:
continue
clone_url = repo["clone_url"]
ref = head["ref"]
if clone_url is None or ref is None:
continue
out.append(f"git+{clone_url}@{ref}")
return out
def test_it_maps_routes(self):
self.mapper.add(b"/route", view)
render = self.mapper.map(
Request(url=URL(path=[u"route"], rooted=True)),
path=b"/route",
)
self.assertIs(render, view)
def assert_valid(response):
# type: (Response) -> None
if response.direct_passthrough:
return
data = response.data
assert isinstance(data, bytes)
# assert response.status_code in [200, 302, 401]
if response.status_code == 302:
return
if URL.from_text(request.url).path in SKIPPED_PATHS:
return
if response.mimetype == "text/html":
assert_html_valid(response)
elif response.mimetype == "application/json":
assert_json_valid(response)
else:
raise AssertionError("Unknown mime type: " + response.mimetype)
return
def requestFromBytes(data=b""):
# type: (bytes) -> FrozenHTTPRequest
return FrozenHTTPRequest(
method=u"GET",
uri=URL.fromText(u"https://twistedmatrix.com/"),
headers=FrozenHTTPHeaders(rawHeaders=()),
body=data,
)
def requestRedirectError(
app: Any, request: IRequest, failure: Failure
) -> KleinRenderable:
"""
Redirect.
"""
url = URL.fromText(failure.value.args[0])
return redirect(request, url)
def test_flash(self):
self.request = request.Request(url=URL(path=[u""]))
self.assertEqual(self.request.messages, [])
self.request.flash(b"Hello World")
self.assertEqual(
self.request.messages,
[request._Message(content=b"Hello World")],
)
def test_it_dumps_practical_json_for_machines(self):
content = ["a", "b", "c"]
render = renderers.bind(renderers.JSON(), to=lambda _: content)
request = Request(
url=URL(path=[u""]),
headers=Headers([("Accept", ["application/json"])]),
)
self.assertNotPretty(content, render(request))
def requestFromBytes(data=b""):
# type: (bytes) -> FrozenHTTPRequest
return FrozenHTTPRequest(
method=u"GET",
uri=URL.fromText(u"https://twistedmatrix.com/"),
headers=FrozenHTTPHeaders(rawHeaders=()),
body=data,
)
def test_dict_sort_keys(self):
content = OrderedDict([("foo", "bar"), ("baz", "quux")])
render = renderers.bind(renderers.JSON(), to=lambda _: content)
request = Request(
url=URL(path=[u""]),
headers=Headers([("Accept", ["*/*"])]),
)
self.assertPretty(content, render(request))
def test_it_dumps_pretty_json_for_humans(self):
content = ["a", "b", "c"]
render = renderers.bind(renderers.JSON(), to=lambda _: content)
request = Request(
url=URL(path=[u""]),
headers=Headers([("Accept", ["*/*"])]),
)
self.assertPretty(content, render(request))
def _reconstitute(self):
"""
Reconstitute this L{URLPath} from all its given attributes.
"""
urltext = urlquote(
urlparse.urlunsplit((self._scheme, self._netloc,
self._path, self._query, self._fragment)),
safe=_allascii
)
self._url = _URL.fromText(urltext.encode("ascii").decode("ascii"))
def _watch_url(self, embed_url):
"""
Generate the URL of the YouTube page at which the embedded video can be
viewed.
"""
video_id = embed_url.path[1]
watch_url = URL(
scheme='https',
host='www.youtube.com',
path=('watch',),
query=(('v', video_id),),
)
try:
[start] = embed_url.get('start')
except (KeyError, ValueError):
return watch_url
if not start.isdigit():
return watch_url # Ignore an invalid second offset.
return watch_url.replace(fragment='t={}s'.format(start))
def __iter__(self, _SRC_ATTR=(None, 'src'), _youtube_hosts=('youtube.com',
'www.youtube.com',
'youtube-nocookie.com',
'www.youtube-nocookie.com')):
html_ns = namespaces['html']
elide = False
for token in BaseFilter.__iter__(self):
token_type = token['type']
if elide:
# NOTE html5lib doesn't permit nesting <iframe> tags,
# (presumably because HTML5 doesn't permit it). Therefore we
# don't need to deal with that case here, just wait for the
# first end tag.
if token_type == 'EndTag' and token['name'] == 'iframe':
elide = False
else:
if (
token_type == 'StartTag' and
token['name'] == 'iframe' and
token['namespace'] == html_ns and
'data' in token and
_SRC_ATTR in token['data']
):
url = URL.from_text(token['data'][_SRC_ATTR])
if url.absolute and url.host in _youtube_hosts and len(url.path) == 2 and url.path[0] == 'embed':
yield {
'type': 'StartTag',
'namespace': html_ns,
'name': 'a',
'data': OrderedDict([
((None, 'href'), self._watch_url(url).to_text()),
]),
}
yield {
'type': 'EmptyTag',
'namespace': html_ns,
'name': u'img',
'data': OrderedDict([
((None, 'alt'), 'YouTube video'),
(_SRC_ATTR, self._thumbnail_url(url).to_text()),
((None, 'width'), '320'),
((None, 'height'), '180'),
]),
}
yield {
'type': 'EndTag',
'namespace': html_ns,
'name': 'a',
}
elide = True
else:
yield token
else:
yield token
def test_converge_complete(self):
"""
At the end of a convergence iteration, ``_CONVERGE_COMPLETE`` is updated
to the current time.
"""
interval = 45
reactor = MemoryReactorClock()
deploy_config = DeploymentConfiguration(
domain=u"s4.example.com",
kubernetes_namespace=u"testing",
subscription_manager_endpoint=URL.from_text(u"http://localhost:8000"),
s3_access_key_id=u"access key id",
s3_secret_key=u"secret key",
introducer_image=u"introducer:abcdefgh",
storageserver_image=u"storageserver:abcdefgh",
)
state_path = FilePath(self.mktemp().decode("ascii"))
state_path.makedirs()
subscription_client = memory_client(
state_path,
deploy_config.domain,
)
k8s_client = KubeClient(k8s=memory_kubernetes().client())
aws_region = FakeAWSServiceRegion(
access_key=deploy_config.s3_access_key_id,
secret_key=deploy_config.s3_secret_key,
)
d = aws_region.get_route53_client().create_hosted_zone(
u"foo", deploy_config.domain,
)
self.successResultOf(d)
service = _convergence_service(
reactor,
interval,
deploy_config,
subscription_client,
k8s_client,
aws_region,
)
service.startService()
reactor.advance(interval)
last_completed = next(iter(list(
metric.samples[-1][-1]
for metric
in REGISTRY.collect()
if metric.name == u"s4_last_convergence_succeeded"
)))
self.assertThat(reactor.seconds(), Equals(last_completed))
def redirect(
request: IRequest, location: URL, origin: Optional[str] = None
) -> KleinRenderable:
"""
Perform a redirect.
"""
if origin is not None:
try:
location = location.set(origin, request.uri.decode("utf-8"))
except ValueError:
return badRequestResponse(request, "Invalid origin URI")
log.debug(
"Redirect {source} -> {destination}",
source=request.uri.decode("utf-8"), destination=location.asText(),
)
url = location.asText().encode("utf-8")
request.setHeader(HeaderName.contentType.value, ContentType.html.value)
request.setHeader(HeaderName.location.value, url)
request.setResponseCode(http.FOUND)
return RedirectPage(location=location)
def test_initInvalidBodyType(self):
# type: () -> None
"""
L{FrozenHTTPRequest} raises L{TypeError} when given a body of an
unknown type.
"""
e = self.assertRaises(
TypeError,
FrozenHTTPRequest,
method=u"GET",
uri=URL.fromText(u"https://twistedmatrix.com/"),
headers=FrozenHTTPHeaders(rawHeaders=()),
body=object(),
)
self.assertEqual(str(e), "body must be bytes or IFount")
def fromString(klass, url):
"""
Make a L{URLPath} from a L{str} or L{unicode}.
@param url: A L{str} representation of a URL.
@type url: L{str} or L{unicode}.
@return: a new L{URLPath} derived from the given string.
@rtype: L{URLPath}
"""
if not isinstance(url, (str, unicode)):
raise ValueError("'url' must be a str or unicode")
if isinstance(url, bytes):
# On Python 2, accepting 'str' (for compatibility) means we might
# get 'bytes'. On py3, this will not work with bytes due to the
# check above.
return klass.fromBytes(url)
return klass._fromURL(_URL.fromText(url))
async def loginSubmit(self, request: IRequest) -> KleinRenderable:
"""
Endpoint for a login form submission.
"""
username = queryValue(request, "username")
password = queryValue(request, "password", default="")
if username is None:
user = None
else:
user = await self.config.authProvider.lookupUserName(username)
if user is None:
self._log.debug(
"Login failed: no such user: {username}", username=username
)
else:
if password is None:
return invalidQueryResponse(request, "password")
authenticated = await self.config.authProvider.verifyCredentials(
user, password
)
if authenticated:
session = request.getSession()
session.user = user
url = queryValue(request, "o")
if url is None:
location = URLs.app # Default to application home
else:
location = URL.fromText(url)
return redirect(request, location)
else:
self._log.debug(
"Login failed: incorrect credentials for user: {user}",
user=user
)
return self.login(request, failed=True)
def assert_valid(response):
# type: (Response) -> None
if response.direct_passthrough:
return
data = response.data
assert isinstance(data, bytes)
# assert response.status_code in [200, 302, 401]
if response.status_code == 302:
return
if URL.from_text(request.url).path in SKIPPED_PATHS:
return
if response.mimetype == "text/html":
assert_html_valid(response)
elif response.mimetype == "application/json":
assert_json_valid(response)
else:
raise AssertionError(f"Unknown mime type: {response.mimetype}")
def path_from_url(url):
url = str(url)
return "/" + "/".join(URL.from_text(url).path)
def _unprefix(url: URL) -> URL:
prefix = URLs.api.path[:-1]
assert url.path[:len(prefix)] == prefix, (url.path[len(prefix):], prefix)
return url.replace(path=url.path[len(prefix):])