def uri(self): # type: () -> URL request = self._request # This code borrows from t.w.server.Request._prePathURL. if request.isSecure(): scheme = u"https" else: scheme = u"http" netloc = nativeString(request.getRequestHostname()) port = request.getHost().port if request.isSecure(): default = 443 else: default = 80 if port != default: netloc += u":{}".format(port) path = nativeString(request.uri) if path and path[0] == u"/": path = path[1:] return URL.fromText(u"{}://{}/{}".format(scheme, netloc, path))
def requestRedirectError(app: Any, request: IRequest, failure: Failure) -> KleinRenderable: """ Redirect. """ url = URL.fromText(failure.value.args[0]) return redirect(request, url)
def requestFromBytes(data=b""): # type: (bytes) -> FrozenHTTPRequest return FrozenHTTPRequest( method=u"GET", uri=URL.fromText(u"https://twistedmatrix.com/"), headers=FrozenHTTPHeaders(rawHeaders=()), body=data, )
def requestRedirectError( app: Any, request: IRequest, failure: Failure ) -> KleinRenderable: """ Redirect. """ url = URL.fromText(failure.value.args[0]) return redirect(request, url)
def _reconstitute(self): """ Reconstitute this L{URLPath} from all its given attributes. """ urltext = urlquote(urlparse.urlunsplit( (self._scheme, self._netloc, self._path, self._query, self._fragment)), safe=_allascii) self._url = _URL.fromText(urltext.encode("ascii").decode("ascii"))
def _reconstitute(self): """ Reconstitute this L{URLPath} from all its given attributes. """ urltext = urlquote( urlparse.urlunsplit((self._scheme, self._netloc, self._path, self._query, self._fragment)), safe=_allascii ) self._url = _URL.fromText(urltext.encode("ascii").decode("ascii"))
def fromString(klass, url): """ Make a L{URLPath} from a L{str} or L{unicode}. @param url: A L{str} representation of a URL. @type url: L{str} or L{unicode}. @return: a new L{URLPath} derived from the given string. @rtype: L{URLPath} """ if not isinstance(url, str): raise ValueError("'url' must be a str") return klass._fromURL(_URL.fromText(url))
async def _handleUnauthorizedResponse(self, url: URL, response: IResponse) -> None: """ Handle an UNAUTHORIZED response. """ challengeValues = response.headers.getRawHeaders("WWW-Authenticate") if not challengeValues: raise ProtocolError( url, "UNAUTHORIZED response with no WWW-Authenticate header") challengeValue = challengeValues[-1] if challengeValue.startswith("Bearer "): challengeParams = { k: v[1:-1] if v.startswith('"') and v.endswith('"') else v for k, v in (token.split("=") for token in challengeValue[7:].split(",")) } try: realmText = challengeParams["realm"] except KeyError: raise ProtocolError(url, "WWW-Authenticate header with no realm") realm = URL.fromText(realmText) try: service = challengeParams["service"] except KeyError: raise ProtocolError(url, "WWW-Authenticate header with no service") error = challengeParams.get("error", None) if error is not None: message = await response.text() self.log.error( "got error ({error}) in auth challenge: {message}", error=error, message=message, ) else: raise ProtocolError( url, f"WWW-Authenticate header with unknown mechanism: " f"{challengeValue}", ) await self._getAuthToken(realm, service)
async def bootstrapResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for Bootstrap. """ requestURL = URL.fromText(request.uri.decode("ascii")) # Remove URL prefix names = requestURL.path[len(URLs.bootstrapBase.path) - 1:] request.setHeader(HeaderName.contentType.value, ContentType.css.value) return await self.cachedZippedResource(request, self.bootstrapSourceURL, self.bootstrapVersion, self.bootstrapVersion, *names)
def test_initInvalidBodyType(self): # type: () -> None """ L{FrozenHTTPRequest} raises L{TypeError} when given a body of an unknown type. """ e = self.assertRaises( TypeError, FrozenHTTPRequest, method=u"GET", uri=URL.fromText(u"https://twistedmatrix.com/"), headers=FrozenHTTPHeaders(rawHeaders=()), body=object(), ) self.assertEqual(str(e), "body must be bytes or IFount")
async def printBuilds(reactor, url): parsed = URL.fromText(url) anchor, line = parsed.fragment.rsplit('R', 1) owner, repo = parsed.path[:2] pullRequestNumber = parsed.path[-2] githubToken = await secretly( reactor, action=lambda token: token, system="codecov-forensics-github", ) tip = await tipOfBranch(reactor, githubToken, pullRequestNumber) path = await anchorToPath(treq, url, anchor) builds = await buildsWithFileAndLine(treq, owner, repo, tip, path, line) for build in builds: print(build)
def fromString(klass, url): """ Make a L{URLPath} from a L{str} or L{unicode}. @param url: A L{str} representation of a URL. @type url: L{str} or L{unicode}. @return: a new L{URLPath} derived from the given string. @rtype: L{URLPath} """ if not isinstance(url, (str, unicode)): raise ValueError("'url' must be a str or unicode") if isinstance(url, bytes): # On Python 2, accepting 'str' (for compatibility) means we might # get 'bytes'. On py3, this will not work with bytes due to the # check above. return klass.fromBytes(url) return klass._fromURL(_URL.fromText(url))
def fromString(klass, url): """ Make a L{URLPath} from a L{str} or L{unicode}. @param url: A L{str} representation of a URL. @type url: L{str} or L{unicode}. @return: a new L{URLPath} derived from the given string. @rtype: L{URLPath} """ if not isinstance(url, str): raise ValueError("'url' must be a str or unicode") if isinstance(url, bytes): # On Python 2, accepting 'str' (for compatibility) means we might # get 'bytes'. On py3, this will not work with bytes due to the # check above. return klass.fromBytes(url) return klass._fromURL(_URL.fromText(url))
async def loginSubmit(self, request: IRequest) -> KleinRenderable: """ Endpoint for a login form submission. """ username = queryValue(request, "username") password = queryValue(request, "password", default="") if username is None: user = None else: user = await self.config.authProvider.lookupUserName(username) if user is None: self._log.debug( "Login failed: no such user: {username}", username=username ) else: if password is None: return invalidQueryResponse(request, "password") authenticated = await self.config.authProvider.verifyCredentials( user, password ) if authenticated: session = request.getSession() session.user = user url = queryValue(request, "o") if url is None: location = URLs.app # Default to application home else: location = URL.fromText(url) return redirect(request, location) else: self._log.debug( "Login failed: incorrect credentials for user: {user}", user=user ) return self.login(request, failed=True)
class URLs(object): """ Incident Management System URL schema. """ # Main application root: ClassVar = URL.fromText("/") prefix: ClassVar = root.child("ims").child("") urlsJS: ClassVar = prefix.child("urls.js") # Static resources static: ClassVar = prefix.child("static") styleSheet: ClassVar = static.child("style.css") logo: ClassVar = static.child("logo.png") # Auth application auth: ClassVar = prefix.child("auth").child("") login: ClassVar = auth.child("login") logout: ClassVar = auth.child("logout") # External application external: ClassVar = prefix.child("ext").child("") jqueryBase: ClassVar = external.child("jquery").child("") jqueryJS: ClassVar = jqueryBase.child("jquery.min.js") jqueryMap: ClassVar = jqueryBase.child("jquery.min.map") bootstrapBase: ClassVar = external.child("bootstrap").child("") bootstrapCSS: ClassVar = bootstrapBase.child("css", "bootstrap.min.css") bootstrapJS: ClassVar = bootstrapBase.child("js", "bootstrap.min.js") dataTablesBase: ClassVar = external.child("datatables").child("") dataTablesJS: ClassVar = dataTablesBase.child( "media", "js", "jquery.dataTables.min.js" ) dataTablesBootstrapCSS: ClassVar = dataTablesBase.child( "media", "css", "dataTables.bootstrap.min.css" ) dataTablesBootstrapJS: ClassVar = dataTablesBase.child( "media", "js", "dataTables.bootstrap.min.js" ) momentJS: ClassVar = external.child("moment.min.js") lscacheJS: ClassVar = external.child("lscache.min.js") # API application api: ClassVar = prefix.child("api").child("") ping: ClassVar = api.child("ping").child("") acl: ClassVar = api.child("access") streets: ClassVar = api.child("streets") personnel: ClassVar = api.child("personnel").child("") incidentTypes: ClassVar = api.child("incident_types").child("") events: ClassVar = api.child("events").child("") event: ClassVar = events.child("<eventID>").child("") locations: ClassVar = event.child("locations").child("") incidents: ClassVar = event.child("incidents").child("") incidentNumber: ClassVar = incidents.child("<number>") incidentReports: ClassVar = event.child("incident_reports").child("") incidentReport: ClassVar = incidentReports.child("<number>") eventSource: ClassVar = api.child("eventsource") # Web application app: ClassVar = prefix.child("app").child("") imsJS: ClassVar = static.child("ims.js") admin: ClassVar = app.child("admin").child("") adminJS: ClassVar = static.child("admin.js") adminEvents: ClassVar = admin.child("events") adminEventsJS: ClassVar = static.child("admin_events.js") adminIncidentTypes: ClassVar = admin.child("types") adminIncidentTypesJS: ClassVar = static.child("admin_types.js") adminStreets: ClassVar = admin.child("streets") adminStreetsJS: ClassVar = static.child("admin_streets.js") viewEvents: ClassVar = app.child("events").child("") viewEvent: ClassVar = viewEvents.child("<eventID>").child("") viewIncidents: ClassVar = viewEvent.child("incidents").child("") viewIncidentsTemplate: ClassVar = app.child("incidents.html") viewIncidentsJS: ClassVar = static.child("incidents.js") viewIncidentsRelative: ClassVar = URL.fromText("incidents").child("") viewIncidentNumber: ClassVar = viewIncidents.child("<number>") viewIncidentTemplate: ClassVar = app.child("incident.html") viewIncidentJS: ClassVar = static.child("incident.js") viewIncidentReports: ClassVar = ( viewEvent.child("incident_reports").child("") ) viewIncidentReportsTemplate: ClassVar = app.child("incident_reports.html") viewIncidentReportsJS: ClassVar = static.child("incident_reports.js") viewIncidentReportsRelative: ClassVar = ( URL.fromText("incident_reports").child("") ) viewIncidentReportNew: ClassVar = viewIncidentReports.child("new") viewIncidentReportNumber: ClassVar = viewIncidentReports.child("<number>") viewIncidentReportTemplate: ClassVar = app.child("incident_report.html") viewIncidentReportJS: ClassVar = static.child("incident_report.js")
class ExternalApplication(object): """ Application with endpoints for cached external resources. """ _log = Logger() router = Router() config: Configuration = attrib(validator=instance_of(Configuration)) bootstrapVersionNumber = "3.3.7" jqueryVersionNumber = "3.1.0" dataTablesVersionNumber = "1.10.12" momentVersionNumber = "2.14.1" lscacheVersionNumber = "1.0.5" bootstrapVersion = f"bootstrap-{bootstrapVersionNumber}-dist" jqueryVersion = f"jquery-{jqueryVersionNumber}" dataTablesVersion = f"DataTables-{dataTablesVersionNumber}" momentVersion = f"moment-{momentVersionNumber}" lscacheVersion = f"lscache-{lscacheVersionNumber}" bootstrapSourceURL = URL.fromText( f"https://github.com/twbs/bootstrap/releases/download/" f"v{bootstrapVersionNumber}/{bootstrapVersion}.zip") jqueryJSSourceURL = URL.fromText( f"https://code.jquery.com/{jqueryVersion}.min.js") jqueryMapSourceURL = URL.fromText( f"https://code.jquery.com/{jqueryVersion}.min.map") dataTablesSourceURL = URL.fromText( f"https://datatables.net/releases/" f"DataTables-{dataTablesVersionNumber}.zip") momentJSSourceURL = URL.fromText( f"https://cdnjs.cloudflare.com/ajax/libs/moment.js/" f"{momentVersionNumber}/moment.min.js") lscacheJSSourceURL = URL.fromText( f"https://raw.githubusercontent.com/pamelafox/lscache/" f"{lscacheVersionNumber}/lscache.min.js") @router.route(_unprefix(URLs.bootstrapBase), methods=("HEAD", "GET"), branch=True) @static async def bootstrapResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for Bootstrap. """ requestURL = URL.fromText(request.uri.decode("ascii")) # Remove URL prefix names = requestURL.path[len(URLs.bootstrapBase.path) - 1:] request.setHeader(HeaderName.contentType.value, ContentType.css.value) return await self.cachedZippedResource(request, self.bootstrapSourceURL, self.bootstrapVersion, self.bootstrapVersion, *names) @router.route(_unprefix(URLs.jqueryJS), methods=("HEAD", "GET")) @static async def jqueryJSResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for jQuery. """ request.setHeader(HeaderName.contentType.value, ContentType.javascript.value) return await self.cachedResource(request, self.jqueryJSSourceURL, f"{self.jqueryVersion}.min.js") @router.route(_unprefix(URLs.jqueryMap), methods=("HEAD", "GET")) @static async def jqueryMapResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for the jQuery map file. """ request.setHeader(HeaderName.contentType.value, ContentType.json.value) return await self.cachedResource(request, self.jqueryMapSourceURL, f"{self.jqueryVersion}.min.map") @router.route(_unprefix(URLs.dataTablesBase), methods=("HEAD", "GET"), branch=True) @static async def dataTablesResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for DataTables. """ requestURL = URL.fromText(request.uri.decode("ascii")) # Remove URL prefix names = requestURL.path[len(URLs.dataTablesBase.path) - 1:] request.setHeader(HeaderName.contentType.value, ContentType.css.value) return await self.cachedZippedResource(request, self.dataTablesSourceURL, self.dataTablesVersion, self.dataTablesVersion, *names) @router.route(_unprefix(URLs.momentJS), methods=("HEAD", "GET")) @static async def momentJSResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for moment.js. """ request.setHeader(HeaderName.contentType.value, ContentType.javascript.value) return await self.cachedResource(request, self.momentJSSourceURL, f"{self.momentVersion}.min.js") @router.route(_unprefix(URLs.lscacheJS), methods=("HEAD", "GET")) @static async def lscacheJSResource(self, request: IRequest) -> KleinRenderable: """ Endpoint for lscache. """ request.setHeader(HeaderName.contentType.value, ContentType.javascript.value) return await self.cachedResource(request, self.lscacheJSSourceURL, f"{self.lscacheVersion}.min.js") async def cacheFromURL(self, url: URL, name: str) -> Path: """ Download a resource and cache it. """ cacheDir = self.config.CachedResourcesPath cacheDir.mkdir(exist_ok=True) destination = cacheDir / name if not destination.exists(): with NamedTemporaryFile(dir=str(cacheDir), delete=False, suffix=".tmp") as tmp: path = Path(tmp.name) try: await downloadPage(url.asText().encode("utf-8"), tmp) except BaseException as e: self._log.failure("Download failed for {url}: {error}", url=url, error=e) try: path.unlink() except (OSError, IOError) as e: self._log.critical( "Failed to remove temporary file {path}: {error}", path=path, error=e) else: path.rename(destination) return destination async def cachedResource(self, request: IRequest, url: URL, name: str) -> KleinRenderable: """ Retrieve a cached resource. """ path = await self.cacheFromURL(url, name) try: return path.read_bytes() except (OSError, IOError) as e: self._log.error("Unable to open file {path}: {error}", path=path, error=e) return notFoundResponse(request) async def cachedZippedResource( self, request: IRequest, url: URL, archiveName: str, name: str, *names: Any, ) -> KleinRenderable: """ Retrieve a cached resource from a zip file. """ archivePath = await self.cacheFromURL(url, f"{archiveName}.zip") try: filePath = ZipArchive(str(archivePath)) except BadZipfile as e: self._log.error( "Corrupt zip archive {path}: {error}", path=archivePath, error=e, ) try: archivePath.unlink() except (OSError, IOError) as e: self._log.critical( "Failed to remove corrupt zip archive {path}: {error}", path=archivePath, error=e, ) return internalErrorResponse(request) except (OSError, IOError) as e: self._log.critical( "Unable to open zip archive {path}: {error}", path=archivePath, error=e, ) return notFoundResponse(request) filePath = filePath.child(name) for name in names: filePath = filePath.child(name) try: return filePath.getContent() except KeyError: self._log.error( "File not found in ZIP archive: {filePath.path}", filePath=filePath, archive=archivePath, ) return notFoundResponse(request)
def _parse_path(self, full_url): if not isinstance(full_url, six.text_type): full_url = six.text_type(full_url) result = URL.fromText(full_url).to_iri() sequence_identifier, action = result.path return sequence_identifier, action
class URLs(object): """ Incident Management System URL schema. """ # Main application root = URL.fromText("/") prefix = root.child("ims").child("") # Static resources static = prefix.child("static") styleSheet = static.child("style.css") logo = static.child("logo.png") # Auth application auth = prefix.child("auth").child("") login = auth.child("login") logout = auth.child("logout") # External application external = prefix.child("ext").child("") jqueryBase = external.child("jquery").child("") jqueryJS = jqueryBase.child("jquery.min.js") jqueryMap = jqueryBase.child("jquery.min.map") bootstrapBase = external.child("bootstrap").child("") bootstrapCSS = bootstrapBase.child("css", "bootstrap.min.css") bootstrapJS = bootstrapBase.child("js", "bootstrap.min.js") dataTablesBase = external.child("datatables").child("") dataTablesJS = dataTablesBase.child("media", "js", "jquery.dataTables.min.js") dataTablesBootstrapCSS = dataTablesBase.child( "media", "css", "dataTables.bootstrap.min.css") dataTablesBootstrapJS = dataTablesBase.child( "media", "js", "dataTables.bootstrap.min.js") momentJS = external.child("moment.min.js") lscacheJS = external.child("lscache.min.js") # API application api = prefix.child("api").child("") ping = api.child("ping").child("") acl = api.child("access") streets = api.child("streets") personnel = api.child("personnel").child("") incidentTypes = api.child("incident_types").child("") incidentReports = api.child("incident_reports").child("") incidentReport = incidentReports.child("<number>") events = api.child("events").child("") event = events.child("<eventID>").child("") locations = event.child("locations").child("") incidents = event.child("incidents").child("") incidentNumber = incidents.child("<number>") eventSource = api.child("eventsource") # Web application app = prefix.child("app").child("") imsJS = static.child("ims.js") admin = app.child("admin").child("") adminJS = static.child("admin.js") adminAccessControl = admin.child("access") adminAccessControlJS = static.child("admin_access.js") adminIncidentTypes = admin.child("types") adminIncidentTypesJS = static.child("admin_types.js") adminStreets = admin.child("streets") adminStreetsJS = static.child("admin_streets.js") viewEvents = app.child("events").child("") viewEvent = viewEvents.child("<eventID>").child("") viewDispatchQueue = viewEvent.child("queue") viewDispatchQueueTemplate = app.child("queue.html") viewDispatchQueueJS = static.child("queue.js") viewDispatchQueueRelative = URL.fromText("queue") viewIncidents = viewEvent.child("incidents").child("") viewIncidentNumber = viewIncidents.child("<number>") viewIncidentTemplate = app.child("incident.html") viewIncidentJS = static.child("incident.js") viewIncidentReports = app.child("incident_reports").child("") viewIncidentReportsTemplate = app.child("incident_reports.html") viewIncidentReportsJS = static.child("incident_reports.js") viewIncidentReportNew = viewIncidentReports.child("new") viewIncidentReportNumber = viewIncidentReports.child("<number>") viewIncidentReportTemplate = app.child("incident_report.html") viewIncidentReportJS = static.child("incident_report.js")
class Client(object): """ Docker Hub API v2 Client """ # # Class attributes # log: ClassVar[Logger] = Logger() apiVersion: ClassVar[str] = "2" _connectionPool_: ClassVar[Optional[HTTPConnectionPool]] = None defaultRootURL = URL.fromText(dockerHubRegistryURL) @classmethod def main(cls) -> None: """ Command line entry point. """ main() @classmethod def _connectionPool(cls) -> HTTPConnectionPool: if cls._connectionPool_ is None: cls._connectionPool_ = HTTPConnectionPool() return cls._connectionPool_ # # Instance attributes # rootURL: URL = attrib(default=defaultRootURL) _endpoint: Endpoint = attrib(init=False) _auth: Authorization = attrib(factory=Authorization, init=False) def __attrs_post_init__(self) -> None: object.__setattr__( self, "_endpoint", Endpoint(apiVersion=self.apiVersion, root=self.rootURL), ) async def _httpGET(self, url: URL, headers: Headers = emptyHeaders) -> IResponse: from twisted.internet import reactor agent = Agent(reactor, pool=self._connectionPool()) return agent.request(b"GET", url.asText().encode("utf-8"), headers, None) async def _get(self, url: URL) -> IResponse: """ Send a GET request. """ async def get() -> IResponse: headers = Headers({}) if self._auth.token: headers.setRawHeaders("Authorization", [f"Bearer {self._auth.token}"]) self.log.info("GET: {url}\nHeaders: {headers}", url=url, headers=headers) response = await self._httpGET(url, headers=headers) self.log.info( "Response: {response}\nHeaders: {response.headers}", response=response, ) return response response = await (get()) if response.code == UNAUTHORIZED: await self._handleUnauthorizedResponse(url, response) response = await (get()) return response async def _handleUnauthorizedResponse(self, url: URL, response: IResponse) -> None: """ Handle an UNAUTHORIZED response. """ challengeValues = response.headers.getRawHeaders("WWW-Authenticate") if not challengeValues: raise ProtocolError( url, "UNAUTHORIZED response with no WWW-Authenticate header") challengeValue = challengeValues[-1] if challengeValue.startswith("Bearer "): challengeParams = { k: v[1:-1] if v.startswith('"') and v.endswith('"') else v for k, v in (token.split("=") for token in challengeValue[7:].split(",")) } try: realmText = challengeParams["realm"] except KeyError: raise ProtocolError(url, "WWW-Authenticate header with no realm") realm = URL.fromText(realmText) try: service = challengeParams["service"] except KeyError: raise ProtocolError(url, "WWW-Authenticate header with no service") error = challengeParams.get("error", None) if error is not None: message = await response.text() self.log.error( "got error ({error}) in auth challenge: {message}", error=error, message=message, ) else: raise ProtocolError( url, f"WWW-Authenticate header with unknown mechanism: " f"{challengeValue}", ) await self._getAuthToken(realm, service) async def _getAuthToken(self, realm: URL, service: str) -> None: """ Obtain an authorization token from the registry. """ # See https://docs.docker.com/registry/spec/auth/token/ url = realm.set("service", service) self.log.info("Authenticating at {url}...", url=url) response = await self._httpGET(url) json = await response.json() try: self._auth.token = json["token"] except KeyError: raise ProtocolError(realm, "got auth response with no token") # Not captured: # expires_in -> int # issued_at -> date async def ping(self) -> None: """ Check whether the registry host supports the API version in use by the client. """ url = self._endpoint.api self.log.info("Pinging API server at {url}...", url=url) response = await self._get(url) if response.code == OK: return if response.code == NOT_FOUND: raise ProtocolNotSupportedError( url, "server does not support Docker Registry HTTP API V2") if response.code == UNAUTHORIZED: json = await response.json() message = "authorization failed" for error in (Error.fromJSON(e) for e in json.get("errors", [])): self.log.error("Error ({error.code.name}): {error.message}", error=error) if error.code is ErrorCode.UNAUTHORIZED: message = f"{message}: {error.message}" raise ProtocolError(url, message)