def test_url_update(self): url = Url("http://test:123/toto?map=123#456") url.hostname = "example" assert url.netloc == "example:123" assert url.port == 123 assert url.url() == "http://example:123/toto?map=123#456" url = Url("http://test:123/toto?map=123#456") url.netloc = "example" assert url.hostname == "example" assert url.port is None assert url.url() == "http://example/toto?map=123#456" url = Url("http://test:123/toto?map=123#456") url.netloc = "example:234" assert url.hostname == "example" assert url.port == 234 assert url.url() == "http://example:234/toto?map=123#456" url = Url("http://test:123/toto?map=123#456") url.port = 345 assert url.netloc == "test:345" assert url.hostname == "test" assert url.url() == "http://test:345/toto?map=123#456" url = Url("http://test:123/toto?map=123#456") url.port = None assert url.netloc == "test" assert url.hostname == "test" assert url.url() == "http://test/toto?map=123#456"
def report_get(self) -> pyramid.response.Response: """ Get the PDF. """ url = Url("{0!s}/report/{1!s}".format( self.config["print_url"], self.request.matchdict.get("ref"))) if self.config.get("print_get_redirect", False): raise HTTPFound(location=url.url()) return self._proxy_response("print", url)
def filter_capabilities(content: str, wms: bool, url: Url, headers: Dict[str, str], request: pyramid.request.Request) -> str: """Filter the WMS/WFS capabilities.""" wms_structure_ = wms_structure(url, headers.get("Host"), request) ogc_server_ids = (get_ogc_server_wms_url_ids(request) if wms else get_ogc_server_wfs_url_ids(request)).get(url.url()) gmf_private_layers = copy.copy(get_private_layers(ogc_server_ids)) for id_ in list(get_protected_layers(request, ogc_server_ids).keys()): if id_ in gmf_private_layers: del gmf_private_layers[id_] private_layers = set() for gmf_layer in list(gmf_private_layers.values()): for ogc_layer in gmf_layer.layer.split(","): private_layers.add(ogc_layer) if ogc_layer in wms_structure_: private_layers.update(wms_structure_[ogc_layer]) parser = defusedxml.expatreader.create_parser(forbid_external=False) # skip inclusion of DTDs parser.setFeature(xml.sax.handler.feature_external_ges, False) parser.setFeature(xml.sax.handler.feature_external_pes, False) result = StringIO() downstream_handler = XMLGenerator(result, "utf-8") filter_handler = _CapabilitiesFilter(parser, downstream_handler, "Layer" if wms else "FeatureType", layers_blacklist=private_layers) filter_handler.parse(StringIO(content)) # type: ignore return result.getvalue()
def report_get(self) -> pyramid.response.Response: """Get the PDF.""" url = Url( f"{self.request.get_organization_print_url()}/report/{self.request.matchdict.get('ref')}" ) if self.request.registry.settings.get("print_get_redirect", False): raise HTTPFound(location=url.url()) return self._proxy_response("print", url)
def filter_wfst_capabilities(content: str, wfs_url: Url, request: pyramid.request.Request) -> str: """Filter the WTS capabilities.""" writable_layers: Set[str] = set() ogc_server_ids = get_ogc_server_wfs_url_ids(request).get(wfs_url.url()) for gmf_layer in list( get_writable_layers(request, ogc_server_ids).values()): writable_layers |= set(gmf_layer.layer.split(",")) parser = defusedxml.expatreader.create_parser(forbid_external=False) # skip inclusion of DTDs parser.setFeature(xml.sax.handler.feature_external_ges, False) parser.setFeature(xml.sax.handler.feature_external_pes, False) result = StringIO() downstream_handler = XMLGenerator(result, "utf-8") filter_handler = _CapabilitiesFilter(parser, downstream_handler, "FeatureType", layers_whitelist=writable_layers) filter_handler.parse(StringIO(content)) # type: ignore return result.getvalue()
def get_report(self) -> pyramid.response.Response: self.layername = self.request.matchdict["layername"] layer_config = self.config["layers"].get(self.layername) multiple = layer_config.get("multiple", False) ids = self.request.matchdict["ids"] if multiple: ids = ids.split(",") if layer_config is None: raise HTTPBadRequest("Layer not found") features_ids = ([self.layername + "." + id_ for id_ in ids] if multiple else [self.layername + "." + ids]) if layer_config["check_credentials"]: # FIXME: support of mapserver groups ogc_server = (models.DBSession.query(main.OGCServer).filter( main.OGCServer.name == layer_config["ogc_server"]).one()) ogc_server_ids = [ogc_server] private_layers_object = get_private_layers(ogc_server_ids) private_layers_names = [ private_layers_object[oid].name for oid in private_layers_object ] protected_layers_object = get_protected_layers( self.request.user, ogc_server_ids) protected_layers_names = [ protected_layers_object[oid].name for oid in protected_layers_object ] if self.layername in private_layers_names and self.layername not in protected_layers_names: raise HTTPForbidden srs = layer_config["srs"] mapserv_url = self.request.route_url( "mapserverproxy", _query={"ogcserver": layer_config["ogc_server"]}) url = Url(mapserv_url) url.add_query({ "service": "WFS", "version": "1.1.0", "outputformat": "gml3", "request": "GetFeature", "typeName": self.layername, "featureid": ",".join(features_ids), "srsName": srs, }) vector_request_url = url.url() spec = layer_config["spec"] if spec is None: spec = { "layout": self.layername, "outputFormat": "pdf", "attributes": { "ids": [{ "id": id_ } for id_ in ids] } if multiple else { "id": id }, } map_config = layer_config.get("map") if map_config is not None: spec["attributes"]["map"] = self._build_map( mapserv_url, vector_request_url, srs, map_config) maps_config = layer_config.get("maps") if maps_config is not None: spec["attributes"]["maps"] = [] for map_config in maps_config: spec["attributes"]["maps"].append( self._build_map(mapserv_url, vector_request_url, srs, map_config)) else: datasource = layer_config.get("datasource", True) if multiple and datasource: data = dumps(layer_config["data"]) data_list = [ loads( data % { "layername": self.layername, "id": id_, "srs": srs, "mapserv_url": mapserv_url, "vector_request_url": vector_request_url, }) for id_ in ids ] self.walker(spec, "%(datasource)s", data_list) spec = loads( dumps(spec) % { "layername": self.layername, "srs": srs, "mapserv_url": mapserv_url, "vector_request_url": vector_request_url, }) elif multiple: spec = loads( dumps(spec) % { "layername": self.layername, "ids": ",".join(ids), "srs": srs, "mapserv_url": mapserv_url, "vector_request_url": vector_request_url, }) else: spec = loads( dumps(spec) % { "layername": self.layername, "id": ids, "srs": srs, "mapserv_url": mapserv_url, "vector_request_url": vector_request_url, }) return self._do_print(spec)
def _proxy( self, url: Url, params: Dict[str, str] = None, method: str = None, cache: bool = False, body: bytes = None, headers: Dict[str, str] = None, ) -> pyramid.response.Response: # Get query string params = dict(self.request.params) if params is None else params url = url.clone().add_query(params, True) LOG.debug("Send query to URL:\n%s.", url) if method is None: method = self.request.method if headers is None: headers = dict(self.request.headers) # Forward request to target (without Host Header) if url.hostname not in self.host_forward_host and "Host" in headers: headers.pop("Host") # Forward the request tracking ID to the other service. This will allow to follow the logs belonging # to a single request coming from the user headers.setdefault("X-Request-ID", self.request.c2c_request_id) # If we releay want to respect the specification, we should chain with the content of the previous # proxy, see also: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded forwarded = { "for": self.request.client_addr, "proto": self.request.scheme } if "Host" in self.request.headers: forwarded["host"] = self.request.headers["Host"] forwarded_str = ";".join(["=".join(e) for e in forwarded.items()]) if "Forwarded" in headers: headers["Forwarded"] = ",".join( [headers["Forwarded"], forwarded_str]) else: headers["Forwarded"] = forwarded_str if not cache: headers["Cache-Control"] = "no-cache" if method in ("POST", "PUT") and body is None: body = self.request.body try: if method in ("POST", "PUT"): response = requests.request(method, url.url(), data=body, headers=headers, **self.http_options) else: response = requests.request(method, url.url(), headers=headers, **self.http_options) except Exception: errors = [ "Error '%s' while getting the URL:", "%s", "Method: %s", "--- With headers ---", "%s" ] args1 = [ sys.exc_info()[0], url, method, "\n".join([ "{}: {}".format( h, v if h not in ("Authorization", "Cookies") else "***") for h, v in list(headers.items()) ]), ] if method in ("POST", "PUT") and body is not None: errors += ["--- Query with body ---", "%s"] args1.append(body.decode("utf-8")) LOG.error("\n".join(errors), *args1, exc_info=True) raise HTTPBadGateway("Error on backend, See logs for detail") if not response.ok: errors = [ "Error '%s' in response of URL:", "%s", "Status: %d", "Method: %s", "--- With headers ---", "%s", ] args2: List[Union[str, int]] = [ response.reason, url.url(), response.status_code, method, "\n".join([ "{}: {}".format( h, v if h not in ("Authorization", "Cookies") else "***") for h, v in list(headers.items()) ]), ] if method in ("POST", "PUT") and body is not None: errors += ["--- Query with body ---", "%s"] args2.append(body.decode("utf-8")) errors += ["--- Return content ---", "%s"] args2.append(response.text) LOG.error("\n".join(errors), *args2) raise exception_response(response.status_code) return response
def test_url_noparam(self): url = Url("http://example.com/") url.add_query({}) self.assertEqual(url.url(), "http://example.com/")
def _proxy( self, url: Url, params: Optional[Dict[str, str]] = None, method: Optional[str] = None, cache: bool = False, body: Optional[bytes] = None, headers: Optional[Dict[str, str]] = None, ) -> requests.models.Response: # Get query string params = dict(self.request.params) if params is None else params url = url.clone().add_query(params, True) LOG.debug("Send query to URL:\n%s.", url) if method is None: method = self.request.method headers = dict(self.request.headers if headers is None else headers) # Forward request to target (without Host Header). # The original Host will be added back by pyramid. if url.hostname not in self.host_forward_host and "Host" in headers: headers.pop("Host") # Forward the request tracking ID to the other service. This will allow to follow the logs belonging # to a single request coming from the user headers.setdefault("X-Request-ID", self.request.c2c_request_id) # If we really want to respect the specification, we should chain with the content of the previous # proxy, see also: # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded forwarded = { "for": self.request.client_addr, "proto": self.request.scheme } if "Host" in self.request.headers: forwarded["host"] = self.request.headers["Host"] forwarded_str = ";".join(["=".join(e) for e in forwarded.items()]) if "Forwarded" in headers: headers["Forwarded"] = ",".join( [headers["Forwarded"], forwarded_str]) else: headers["Forwarded"] = forwarded_str # Set alternative "X-Forwarded" headers for forwarded_elements in reversed(headers["Forwarded"].split(",")): for element in forwarded_elements.split(";"): key, value = element.split("=") header_key = f"X-Forwarded-{key.capitalize()}" header_value = headers.get(header_key) headers[ header_key] = value if header_value is None else ", ".join( [header_value, value]) if not cache: headers["Cache-Control"] = "no-cache" if method in ("POST", "PUT") and body is None: body = self.request.body headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist) try: if method in ("POST", "PUT"): response = requests.request(method, url.url(), data=body, headers=headers, **self.http_options) else: response = requests.request(method, url.url(), headers=headers, **self.http_options) except Exception: errors = [ "Error '%s' while getting the URL:", "%s", "Method: %s", "--- With headers ---", "%s" ] args1 = [ sys.exc_info()[0], url, method, "\n".join([ f"{h}: {v if h not in ('Authorization', 'Cookies') else '***'}" for h, v in list(headers.items()) ]), ] if method in ("POST", "PUT") and body is not None: errors += ["--- Query with body ---", "%s"] args1.append(body.decode("utf-8")) LOG.exception("\n".join(errors), *args1) raise HTTPBadGateway( # pylint: disable=raise-missing-from "Error on backend, See logs for detail") if not response.ok: errors = [ "Error '%s' in response of URL:", "%s", "Status: %d", "Method: %s", "--- With headers ---", "%s", ] args2: List[Union[str, int]] = [ response.reason, url.url(), response.status_code, method, "\n".join([ f"{h}: {v if h not in ('Authorization', 'Cookies') else '***'}" for h, v in list(headers.items()) ]), ] if method in ("POST", "PUT") and body is not None: errors += ["--- Query with body ---", "%s"] args2.append(body.decode("utf-8")) errors += ["--- Return content ---", "%s"] args2.append(response.text) LOG.error("\n".join(errors), *args2) raise exception_response(response.status_code) if not response.headers.get("Content-Type", "").startswith("image/"): LOG.debug("Get result for URL: %s:\n%s.", url, body) return response