def wiki_path(self, slug): url = URL( scheme='https', host='en.wikipedia.org', path=['wiki', str(slug).replace(' ', '_')], ) return url.to_text()
def urls(draw: Callable, collection: Optional[bool] = None) -> URL: """ Strategy that generates URLs. """ urlPathText = partial(text, alphabet=characters(blacklist_characters="/?#"), max_size=32) segments = draw(lists(urlPathText(), max_size=16)) url = URL( scheme=draw(sampled_from(("http", "https"))), # FIXME: wimpy host name alphabet host=draw(text(alphabet=ascii_letters, min_size=1)), port=draw(integers(min_value=1, max_value=65535)), path=segments, ) if collection: url = url.child("") else: if collection is not None: url = url.child(draw(urlPathText(min_size=1))) return url
def test_rooted_empty(self): """ Rooted empty path. """ self.assertThat([ url_path(URL.from_text(u'http://example.com/')), url_path(URL.from_text(u'/')) ], AllMatch(Equals(u'/')))
def test_rooted(self): """ Rooted path. """ self.assertThat([ url_path(URL.from_text(u'http://example.com/foo/bar')), url_path(URL.from_text(u'/foo/bar')) ], AllMatch(Equals(u'/foo/bar')))
def test_known_HTTP_methods_use_associated_renderer(self): render = traversal.method_delegate( GET=lambda request: b"foo", put=lambda request: request.method, ) get = Request(url=URL(path=[u""]), method=b"GET") self.assertEqual(render(get), b"foo") put = Request(url=URL(path=[u""]), method=b"PUT") self.assertEqual(render(put), b"PUT")
def test_it_maps_multiple_routes_for_specified_methods(self): self.mapper.add(b"/", lambda r: Response(b"GET"), methods=[b"GET"]) self.mapper.add(b"/", lambda r: Response(b"POST"), methods=[b"POST"]) get = Request(url=URL(path=[u""], rooted=True), method=b"GET") post = Request(url=URL(path=[u""], rooted=True), method=b"POST") map = self.mapper.map self.assertEqual( (map(get, path=b"/")(get), map(post, path=b"/")(post)), (Response(b"GET"), Response(b"POST")), )
def render(self, twistedRequest): path = twistedRequest.uri.lstrip("/").decode("ascii").split(u"/") request = Request( content=twistedRequest.content, headers=Headers(twistedRequest.requestHeaders.getAllRawHeaders()), method=twistedRequest.method, url=URL( scheme=u"https" if twistedRequest.isSecure() else u"http", host=twistedRequest.getRequestHostname().decode("ascii"), path=path, query=[(k.decode("ascii"), each.decode("ascii")) for k, v in twistedRequest.args.items() for each in v], ), ) response = self.application.serve( request=request, path=b"/" + b"/".join(twistedRequest.postpath), ) twistedRequest.setResponseCode(response.code) for k, v in response.headers.canonicalized(): twistedRequest.responseHeaders.setRawHeaders(k, v) return response.content
def _nevow_request_to_request_map(req): """ Convert a Nevow request object into an immutable request map. """ headers = req.requestHeaders content_type, character_encoding = _get_content_type(headers) iri = URL.from_text(req.uri.decode('utf-8')).to_iri() host = _get_first_header(headers, b'host').decode('utf-8') scheme = u'https' if req.isSecure() else u'http' if u':' in host: host, port = host.split(u':', 1) port = int(port) else: port = {u'https': 443, u'http': 80}.get(scheme) return m( body=req.content, content_type=content_type, content_length=_get_first_header(headers, b'content-length'), character_encoding=character_encoding, headers=freeze(dict(headers.getAllRawHeaders())), remote_addr=req.getClientIP(), request_method=req.method, server_name=host, server_port=port, scheme=scheme, #ssl_client_cert=XXX, uri=iri, #query_string path_info=url_path(iri), protocol=getattr(req, 'clientproto', None))
def retarget_links(html_tree, mode='external'): if mode == 'none': return elif mode not in ('external', 'all'): raise ValueError('expected "none", "external", or "all", not: %r' % mode) for el in html_tree.iter(): if not isinstance(el.tag, basestring): continue if not el.tag == 'a': continue if el.get('target'): continue # let explicit settings lie href = el.get('href') if not href or href.startswith('#'): continue if mode == 'all': retarget = True elif mode == 'external': try: url = URL.from_text(href) except ValueError: retarget = True else: retarget = bool(url.host) if retarget: el.set('target', '_blank') el.set('rel', 'noopener') return
def _a_handler(a_el): href = a_el.attrib.get('href') if href: url = urljoin(a_el.base_url, href) if not isinstance(url, six.text_type): url = six.u(url) return URL.from_text(url)
def setUp(self): service = StubFeed({ URL.from_text(feed._source).host.encode('ascii'): makeXML(feed) for feed in FEEDS }) treq = StubTreq(service.resource()) self.retriever = FeedRetrieval(treq=treq)
def _thumbnail_url(self, embed_url) -> DecodedURL: """ Generate the URL of the thumbnail for a YouTube video embed. There are a few apparent options for the filename in this URL: default.jpg — always seems to be available, even on old videos, but is tiny (120×90). mqdefault.jpg — always seems to be available, and has a more contemporary aspect ratio (320×180). This is what this function returns. hqdefault.jpg — always seems to be available (though I haven't tested videos more than 5 years old), but it is always 4:3 (480×360), so recent videos are letterboxed. maxresdefault.jpg — only recent videos seem to have this (e.g., all of the videos I've checked on the YouTube homepage as of October 2017 do). An old video which lacks maxresdefault.jpg: https://www.youtube.com/watch?v=XPIFncE22pw A recent video from the homepage which has all of them: https://www.youtube.com/watch?v=R1ZXOOLMJ8s See also https://boingboing.net/features/getthumbs.html """ video_id = embed_url.path[1] return DecodedURL( URL( scheme="https", host="i.ytimg.com", path=("vi", video_id, "mqdefault.jpg"), ), )
def render_GET(self, req): """ Historically, accessing this via "GET /uri?uri=<capabilitiy>" was/is a feature -- which simply redirects to the more-common "GET /uri/<capability>" with any other query args preserved. New code should use "/uri/<cap>" """ uri_arg = req.args.get(b"uri", [None])[0] if uri_arg is None: raise WebError("GET /uri requires uri=") # shennanigans like putting "%2F" or just "/" itself, or ../ # etc in the <cap> might be a vector for weirdness so we # validate that this is a valid capability before proceeding. cap = uri.from_string(uri_arg) if isinstance(cap, uri.UnknownURI): raise WebError("Invalid capability") # so, using URL.from_text(req.uri) isn't going to work because # it seems Nevow was creating absolute URLs including # host/port whereas req.uri is absolute (but lacks host/port) redir_uri = URL.from_text(req.prePathURL().decode('utf8')) redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8')) # add back all the query args that AREN'T "?uri=" for k, values in req.args.items(): if k != b"uri": for v in values: redir_uri = redir_uri.add(k.decode('utf8'), v.decode('utf8')) return redirectTo(redir_uri.to_text().encode('utf8'), req)
def test_basic(self): """ Test basic request map keys. """ request = fake_twisted_request(request_headers={ b'x-foo': [b'bar'], }) self.assertThat( _nevow_request_to_request_map(request), ContainsDict({ 'content_type': Equals(b'application/octet-stream'), 'content_length': Equals(0), 'character_encoding': Is(None), 'headers': Equals({ b'Content-Length': [0], b'X-Foo': [b'bar'], b'Host': [b'example.com'] }), 'remote_addr': Equals(b'192.168.1.1'), 'request_method': Equals(b'GET'), 'server_name': Equals(b'example.com'), 'server_port': Equals(80), 'scheme': Equals(b'http'), 'uri': Equals(URL.from_text(u'/one')) }))
def construct( cls, host=None, user=None, path=None, ): # test whether the path is rooted rooted = path.startswith('/') # the path must be rooted when a remote host is given if host is not None and not rooted: raise ValueError( "Path must be a fully rooted path when a host is given") # get the sections of the path path_segs = path.strip('/').split('/') if user is None: user = '' url = URL( scheme='rsync', host=host, userinfo=user, rooted=rooted, path=path_segs, ) return Endpoint(url)
def uri(self): # type: () -> URL request = self._request # This code borrows from t.w.server.Request._prePathURL. if request.isSecure(): scheme = u"https" else: scheme = u"http" netloc = nativeString(request.getRequestHostname()) port = request.getHost().port if request.isSecure(): default = 443 else: default = 80 if port != default: netloc += u":{}".format(port) path = nativeString(request.uri) if path and path[0] == u"/": path = path[1:] return URL.fromText(u"{}://{}/{}".format(scheme, netloc, path))
class TestUnicodeRenderer(TestCase): request = Request(url=URL(path=[u""])) def test_it_renders_via_the_given_encoding(self): renderer = renderers.Unicode(encoding="utf-8") render = renderers.bind(renderer, to=lambda _: u"שלום") self.assertEqual( render(self.request), Response(u"שלום".encode("utf-8")), ) def test_encoding_ignoring_errors(self): renderer = renderers.Unicode(encoding="ascii", errors="ignore") render = renderers.bind(renderer, to=lambda _: u"שלום") self.assertEqual(render(self.request), Response(b"")) def test_encoding_errors_by_default(self): renderer = renderers.Unicode(encoding="ascii") render = renderers.bind(renderer, to=lambda _: u"שלום") with self.assertRaises(UnicodeEncodeError): render(self.request) def test_UTF8(self): render = renderers.bind(renderers.UTF8, to=lambda _: u"שלום") self.assertEqual( render(self.request), Response(u"שלום".encode("utf-8")), )
def requestRedirectError(app: Any, request: IRequest, failure: Failure) -> KleinRenderable: """ Redirect. """ url = URL.fromText(failure.value.args[0]) return redirect(request, url)
def fake_nevow_request(method='GET', body=b'', is_secure=False, uri=b'http://example.com/one', request_headers=None, Request=NevowRequest): """ Create a fake `NevowRequest` instance for the purposes of testing. """ uri = URL.from_text(uri.decode('utf-8')).to_uri() channel = DummyChannel() if is_secure: channel.transport = DummyChannel.SSL() request = Request(channel=channel) request.method = method request.uri = url_path(uri) request.clientproto = b'HTTP/1.1' request.client = channel.transport.getPeer() content_length = len(body) request.requestHeaders.setRawHeaders('host', [uri.authority().encode('utf-8')]) request.requestHeaders.setRawHeaders('content-length', [content_length]) if request_headers: for k, v in request_headers.items(): request.requestHeaders.setRawHeaders(k, v) request.gotLength(content_length) request.content.write(body) request.content.seek(0) return request
async def cacheFromURL(self, url: URL, name: str) -> Path: """ Download a resource and cache it. """ cacheDir = self.config.CachedResourcesPath cacheDir.mkdir(exist_ok=True) destination = cacheDir / name if not destination.exists(): with NamedTemporaryFile(dir=str(cacheDir), delete=False, suffix=".tmp") as tmp: path = Path(tmp.name) try: await downloadPage(url.asText().encode("utf-8"), tmp) except BaseException as e: self._log.failure("Download failed for {url}: {error}", url=url, error=e) try: path.unlink() except (OSError, IOError) as e: self._log.critical( "Failed to remove temporary file {path}: {error}", path=path, error=e) else: path.rename(destination) return destination
def test_specified_renderer_multiple_routes(self): self.router.add(b"/", view, methods=[b"GET"], renderer=None) self.router.add( b"/", view, methods=[b"POST"], renderer=ReverseRenderer(), ) get = Request(url=URL(path=[u""]), method=b"GET") response = self.router.route(get, path=b"/") self.assertEqual(response, Response(b"{}")) post = Request(url=URL(path=[u""]), method=b"POST") response = self.router.route(post, path=b"/") self.assertEqual(response, Response(b"}{"))
def test_it_maps_routes_for_specified_methods(self): self.mapper.add(b"/route", view, methods=[b"POST"]) request = Request( url=URL(path=[u"route"], rooted=True), method=b"POST", ) self.assertEqual(self.mapper.map(request, path=b"/route"), view)
def get_pull_requests(url: hyperlink.URL) -> t.List[str]: url = url.replace(host="api.github.com", path=("repos", ) + url.path + ("pulls", )) r = requests.get(url) r.raise_for_status() pulls = r.json() out = [] assert isinstance(pulls, list) for pull in pulls: head = pull["head"] if head is None: continue repo = head["repo"] if repo is None: continue clone_url = repo["clone_url"] ref = head["ref"] if clone_url is None or ref is None: continue out.append(f"git+{clone_url}@{ref}") return out
def test_it_maps_routes(self): self.mapper.add(b"/route", view) render = self.mapper.map( Request(url=URL(path=[u"route"], rooted=True)), path=b"/route", ) self.assertIs(render, view)
def assert_valid(response): # type: (Response) -> None if response.direct_passthrough: return data = response.data assert isinstance(data, bytes) # assert response.status_code in [200, 302, 401] if response.status_code == 302: return if URL.from_text(request.url).path in SKIPPED_PATHS: return if response.mimetype == "text/html": assert_html_valid(response) elif response.mimetype == "application/json": assert_json_valid(response) else: raise AssertionError("Unknown mime type: " + response.mimetype) return
def requestFromBytes(data=b""): # type: (bytes) -> FrozenHTTPRequest return FrozenHTTPRequest( method=u"GET", uri=URL.fromText(u"https://twistedmatrix.com/"), headers=FrozenHTTPHeaders(rawHeaders=()), body=data, )
def requestRedirectError( app: Any, request: IRequest, failure: Failure ) -> KleinRenderable: """ Redirect. """ url = URL.fromText(failure.value.args[0]) return redirect(request, url)
def test_flash(self): self.request = request.Request(url=URL(path=[u""])) self.assertEqual(self.request.messages, []) self.request.flash(b"Hello World") self.assertEqual( self.request.messages, [request._Message(content=b"Hello World")], )
def test_it_dumps_practical_json_for_machines(self): content = ["a", "b", "c"] render = renderers.bind(renderers.JSON(), to=lambda _: content) request = Request( url=URL(path=[u""]), headers=Headers([("Accept", ["application/json"])]), ) self.assertNotPretty(content, render(request))
def test_dict_sort_keys(self): content = OrderedDict([("foo", "bar"), ("baz", "quux")]) render = renderers.bind(renderers.JSON(), to=lambda _: content) request = Request( url=URL(path=[u""]), headers=Headers([("Accept", ["*/*"])]), ) self.assertPretty(content, render(request))
def test_it_dumps_pretty_json_for_humans(self): content = ["a", "b", "c"] render = renderers.bind(renderers.JSON(), to=lambda _: content) request = Request( url=URL(path=[u""]), headers=Headers([("Accept", ["*/*"])]), ) self.assertPretty(content, render(request))
def _reconstitute(self): """ Reconstitute this L{URLPath} from all its given attributes. """ urltext = urlquote( urlparse.urlunsplit((self._scheme, self._netloc, self._path, self._query, self._fragment)), safe=_allascii ) self._url = _URL.fromText(urltext.encode("ascii").decode("ascii"))
def _watch_url(self, embed_url): """ Generate the URL of the YouTube page at which the embedded video can be viewed. """ video_id = embed_url.path[1] watch_url = URL( scheme='https', host='www.youtube.com', path=('watch',), query=(('v', video_id),), ) try: [start] = embed_url.get('start') except (KeyError, ValueError): return watch_url if not start.isdigit(): return watch_url # Ignore an invalid second offset. return watch_url.replace(fragment='t={}s'.format(start))
def __iter__(self, _SRC_ATTR=(None, 'src'), _youtube_hosts=('youtube.com', 'www.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com')): html_ns = namespaces['html'] elide = False for token in BaseFilter.__iter__(self): token_type = token['type'] if elide: # NOTE html5lib doesn't permit nesting <iframe> tags, # (presumably because HTML5 doesn't permit it). Therefore we # don't need to deal with that case here, just wait for the # first end tag. if token_type == 'EndTag' and token['name'] == 'iframe': elide = False else: if ( token_type == 'StartTag' and token['name'] == 'iframe' and token['namespace'] == html_ns and 'data' in token and _SRC_ATTR in token['data'] ): url = URL.from_text(token['data'][_SRC_ATTR]) if url.absolute and url.host in _youtube_hosts and len(url.path) == 2 and url.path[0] == 'embed': yield { 'type': 'StartTag', 'namespace': html_ns, 'name': 'a', 'data': OrderedDict([ ((None, 'href'), self._watch_url(url).to_text()), ]), } yield { 'type': 'EmptyTag', 'namespace': html_ns, 'name': u'img', 'data': OrderedDict([ ((None, 'alt'), 'YouTube video'), (_SRC_ATTR, self._thumbnail_url(url).to_text()), ((None, 'width'), '320'), ((None, 'height'), '180'), ]), } yield { 'type': 'EndTag', 'namespace': html_ns, 'name': 'a', } elide = True else: yield token else: yield token
def test_converge_complete(self): """ At the end of a convergence iteration, ``_CONVERGE_COMPLETE`` is updated to the current time. """ interval = 45 reactor = MemoryReactorClock() deploy_config = DeploymentConfiguration( domain=u"s4.example.com", kubernetes_namespace=u"testing", subscription_manager_endpoint=URL.from_text(u"http://localhost:8000"), s3_access_key_id=u"access key id", s3_secret_key=u"secret key", introducer_image=u"introducer:abcdefgh", storageserver_image=u"storageserver:abcdefgh", ) state_path = FilePath(self.mktemp().decode("ascii")) state_path.makedirs() subscription_client = memory_client( state_path, deploy_config.domain, ) k8s_client = KubeClient(k8s=memory_kubernetes().client()) aws_region = FakeAWSServiceRegion( access_key=deploy_config.s3_access_key_id, secret_key=deploy_config.s3_secret_key, ) d = aws_region.get_route53_client().create_hosted_zone( u"foo", deploy_config.domain, ) self.successResultOf(d) service = _convergence_service( reactor, interval, deploy_config, subscription_client, k8s_client, aws_region, ) service.startService() reactor.advance(interval) last_completed = next(iter(list( metric.samples[-1][-1] for metric in REGISTRY.collect() if metric.name == u"s4_last_convergence_succeeded" ))) self.assertThat(reactor.seconds(), Equals(last_completed))
def redirect( request: IRequest, location: URL, origin: Optional[str] = None ) -> KleinRenderable: """ Perform a redirect. """ if origin is not None: try: location = location.set(origin, request.uri.decode("utf-8")) except ValueError: return badRequestResponse(request, "Invalid origin URI") log.debug( "Redirect {source} -> {destination}", source=request.uri.decode("utf-8"), destination=location.asText(), ) url = location.asText().encode("utf-8") request.setHeader(HeaderName.contentType.value, ContentType.html.value) request.setHeader(HeaderName.location.value, url) request.setResponseCode(http.FOUND) return RedirectPage(location=location)
def test_initInvalidBodyType(self): # type: () -> None """ L{FrozenHTTPRequest} raises L{TypeError} when given a body of an unknown type. """ e = self.assertRaises( TypeError, FrozenHTTPRequest, method=u"GET", uri=URL.fromText(u"https://twistedmatrix.com/"), headers=FrozenHTTPHeaders(rawHeaders=()), body=object(), ) self.assertEqual(str(e), "body must be bytes or IFount")
def fromString(klass, url): """ Make a L{URLPath} from a L{str} or L{unicode}. @param url: A L{str} representation of a URL. @type url: L{str} or L{unicode}. @return: a new L{URLPath} derived from the given string. @rtype: L{URLPath} """ if not isinstance(url, (str, unicode)): raise ValueError("'url' must be a str or unicode") if isinstance(url, bytes): # On Python 2, accepting 'str' (for compatibility) means we might # get 'bytes'. On py3, this will not work with bytes due to the # check above. return klass.fromBytes(url) return klass._fromURL(_URL.fromText(url))
async def loginSubmit(self, request: IRequest) -> KleinRenderable: """ Endpoint for a login form submission. """ username = queryValue(request, "username") password = queryValue(request, "password", default="") if username is None: user = None else: user = await self.config.authProvider.lookupUserName(username) if user is None: self._log.debug( "Login failed: no such user: {username}", username=username ) else: if password is None: return invalidQueryResponse(request, "password") authenticated = await self.config.authProvider.verifyCredentials( user, password ) if authenticated: session = request.getSession() session.user = user url = queryValue(request, "o") if url is None: location = URLs.app # Default to application home else: location = URL.fromText(url) return redirect(request, location) else: self._log.debug( "Login failed: incorrect credentials for user: {user}", user=user ) return self.login(request, failed=True)
def assert_valid(response): # type: (Response) -> None if response.direct_passthrough: return data = response.data assert isinstance(data, bytes) # assert response.status_code in [200, 302, 401] if response.status_code == 302: return if URL.from_text(request.url).path in SKIPPED_PATHS: return if response.mimetype == "text/html": assert_html_valid(response) elif response.mimetype == "application/json": assert_json_valid(response) else: raise AssertionError(f"Unknown mime type: {response.mimetype}")
def path_from_url(url): url = str(url) return "/" + "/".join(URL.from_text(url).path)
def _unprefix(url: URL) -> URL: prefix = URLs.api.path[:-1] assert url.path[:len(prefix)] == prefix, (url.path[len(prefix):], prefix) return url.replace(path=url.path[len(prefix):])