def test_url_update(self):
url = Url("http://test:123/toto?map=123#456")
url.hostname = "example"
assert url.netloc == "example:123"
assert url.port == 123
assert url.url() == "http://example:123/toto?map=123#456"
url = Url("http://test:123/toto?map=123#456")
url.netloc = "example"
assert url.hostname == "example"
assert url.port is None
assert url.url() == "http://example/toto?map=123#456"
url = Url("http://test:123/toto?map=123#456")
url.netloc = "example:234"
assert url.hostname == "example"
assert url.port == 234
assert url.url() == "http://example:234/toto?map=123#456"
url = Url("http://test:123/toto?map=123#456")
url.port = 345
assert url.netloc == "test:345"
assert url.hostname == "test"
assert url.url() == "http://test:345/toto?map=123#456"
url = Url("http://test:123/toto?map=123#456")
url.port = None
assert url.netloc == "test"
assert url.hostname == "test"
assert url.url() == "http://test/toto?map=123#456"
def report_get(self) -> pyramid.response.Response:
""" Get the PDF. """
url = Url("{0!s}/report/{1!s}".format(
self.config["print_url"], self.request.matchdict.get("ref")))
if self.config.get("print_get_redirect", False):
raise HTTPFound(location=url.url())
return self._proxy_response("print", url)
def filter_capabilities(content: str, wms: bool, url: Url, headers: Dict[str,
str],
request: pyramid.request.Request) -> str:
"""Filter the WMS/WFS capabilities."""
wms_structure_ = wms_structure(url, headers.get("Host"), request)
ogc_server_ids = (get_ogc_server_wms_url_ids(request) if wms else
get_ogc_server_wfs_url_ids(request)).get(url.url())
gmf_private_layers = copy.copy(get_private_layers(ogc_server_ids))
for id_ in list(get_protected_layers(request, ogc_server_ids).keys()):
if id_ in gmf_private_layers:
del gmf_private_layers[id_]
private_layers = set()
for gmf_layer in list(gmf_private_layers.values()):
for ogc_layer in gmf_layer.layer.split(","):
private_layers.add(ogc_layer)
if ogc_layer in wms_structure_:
private_layers.update(wms_structure_[ogc_layer])
parser = defusedxml.expatreader.create_parser(forbid_external=False)
# skip inclusion of DTDs
parser.setFeature(xml.sax.handler.feature_external_ges, False)
parser.setFeature(xml.sax.handler.feature_external_pes, False)
result = StringIO()
downstream_handler = XMLGenerator(result, "utf-8")
filter_handler = _CapabilitiesFilter(parser,
downstream_handler,
"Layer" if wms else "FeatureType",
layers_blacklist=private_layers)
filter_handler.parse(StringIO(content)) # type: ignore
return result.getvalue()
def report_get(self) -> pyramid.response.Response:
"""Get the PDF."""
url = Url(
f"{self.request.get_organization_print_url()}/report/{self.request.matchdict.get('ref')}"
)
if self.request.registry.settings.get("print_get_redirect", False):
raise HTTPFound(location=url.url())
return self._proxy_response("print", url)
def filter_wfst_capabilities(content: str, wfs_url: Url,
request: pyramid.request.Request) -> str:
"""Filter the WTS capabilities."""
writable_layers: Set[str] = set()
ogc_server_ids = get_ogc_server_wfs_url_ids(request).get(wfs_url.url())
for gmf_layer in list(
get_writable_layers(request, ogc_server_ids).values()):
writable_layers |= set(gmf_layer.layer.split(","))
parser = defusedxml.expatreader.create_parser(forbid_external=False)
# skip inclusion of DTDs
parser.setFeature(xml.sax.handler.feature_external_ges, False)
parser.setFeature(xml.sax.handler.feature_external_pes, False)
result = StringIO()
downstream_handler = XMLGenerator(result, "utf-8")
filter_handler = _CapabilitiesFilter(parser,
downstream_handler,
"FeatureType",
layers_whitelist=writable_layers)
filter_handler.parse(StringIO(content)) # type: ignore
return result.getvalue()
def get_report(self) -> pyramid.response.Response:
self.layername = self.request.matchdict["layername"]
layer_config = self.config["layers"].get(self.layername)
multiple = layer_config.get("multiple", False)
ids = self.request.matchdict["ids"]
if multiple:
ids = ids.split(",")
if layer_config is None:
raise HTTPBadRequest("Layer not found")
features_ids = ([self.layername + "." + id_ for id_ in ids]
if multiple else [self.layername + "." + ids])
if layer_config["check_credentials"]:
# FIXME: support of mapserver groups
ogc_server = (models.DBSession.query(main.OGCServer).filter(
main.OGCServer.name == layer_config["ogc_server"]).one())
ogc_server_ids = [ogc_server]
private_layers_object = get_private_layers(ogc_server_ids)
private_layers_names = [
private_layers_object[oid].name
for oid in private_layers_object
]
protected_layers_object = get_protected_layers(
self.request.user, ogc_server_ids)
protected_layers_names = [
protected_layers_object[oid].name
for oid in protected_layers_object
]
if self.layername in private_layers_names and self.layername not in protected_layers_names:
raise HTTPForbidden
srs = layer_config["srs"]
mapserv_url = self.request.route_url(
"mapserverproxy", _query={"ogcserver": layer_config["ogc_server"]})
url = Url(mapserv_url)
url.add_query({
"service": "WFS",
"version": "1.1.0",
"outputformat": "gml3",
"request": "GetFeature",
"typeName": self.layername,
"featureid": ",".join(features_ids),
"srsName": srs,
})
vector_request_url = url.url()
spec = layer_config["spec"]
if spec is None:
spec = {
"layout": self.layername,
"outputFormat": "pdf",
"attributes": {
"ids": [{
"id": id_
} for id_ in ids]
} if multiple else {
"id": id
},
}
map_config = layer_config.get("map")
if map_config is not None:
spec["attributes"]["map"] = self._build_map(
mapserv_url, vector_request_url, srs, map_config)
maps_config = layer_config.get("maps")
if maps_config is not None:
spec["attributes"]["maps"] = []
for map_config in maps_config:
spec["attributes"]["maps"].append(
self._build_map(mapserv_url, vector_request_url, srs,
map_config))
else:
datasource = layer_config.get("datasource", True)
if multiple and datasource:
data = dumps(layer_config["data"])
data_list = [
loads(
data % {
"layername": self.layername,
"id": id_,
"srs": srs,
"mapserv_url": mapserv_url,
"vector_request_url": vector_request_url,
}) for id_ in ids
]
self.walker(spec, "%(datasource)s", data_list)
spec = loads(
dumps(spec) % {
"layername": self.layername,
"srs": srs,
"mapserv_url": mapserv_url,
"vector_request_url": vector_request_url,
})
elif multiple:
spec = loads(
dumps(spec) % {
"layername": self.layername,
"ids": ",".join(ids),
"srs": srs,
"mapserv_url": mapserv_url,
"vector_request_url": vector_request_url,
})
else:
spec = loads(
dumps(spec) % {
"layername": self.layername,
"id": ids,
"srs": srs,
"mapserv_url": mapserv_url,
"vector_request_url": vector_request_url,
})
return self._do_print(spec)
def _proxy(
self,
url: Url,
params: Dict[str, str] = None,
method: str = None,
cache: bool = False,
body: bytes = None,
headers: Dict[str, str] = None,
) -> pyramid.response.Response:
# Get query string
params = dict(self.request.params) if params is None else params
url = url.clone().add_query(params, True)
LOG.debug("Send query to URL:\n%s.", url)
if method is None:
method = self.request.method
if headers is None:
headers = dict(self.request.headers)
# Forward request to target (without Host Header)
if url.hostname not in self.host_forward_host and "Host" in headers:
headers.pop("Host")
# Forward the request tracking ID to the other service. This will allow to follow the logs belonging
# to a single request coming from the user
headers.setdefault("X-Request-ID", self.request.c2c_request_id)
# If we releay want to respect the specification, we should chain with the content of the previous
# proxy, see also:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
forwarded = {
"for": self.request.client_addr,
"proto": self.request.scheme
}
if "Host" in self.request.headers:
forwarded["host"] = self.request.headers["Host"]
forwarded_str = ";".join(["=".join(e) for e in forwarded.items()])
if "Forwarded" in headers:
headers["Forwarded"] = ",".join(
[headers["Forwarded"], forwarded_str])
else:
headers["Forwarded"] = forwarded_str
if not cache:
headers["Cache-Control"] = "no-cache"
if method in ("POST", "PUT") and body is None:
body = self.request.body
try:
if method in ("POST", "PUT"):
response = requests.request(method,
url.url(),
data=body,
headers=headers,
**self.http_options)
else:
response = requests.request(method,
url.url(),
headers=headers,
**self.http_options)
except Exception:
errors = [
"Error '%s' while getting the URL:", "%s", "Method: %s",
"--- With headers ---", "%s"
]
args1 = [
sys.exc_info()[0],
url,
method,
"\n".join([
"{}: {}".format(
h,
v if h not in ("Authorization", "Cookies") else "***")
for h, v in list(headers.items())
]),
]
if method in ("POST", "PUT") and body is not None:
errors += ["--- Query with body ---", "%s"]
args1.append(body.decode("utf-8"))
LOG.error("\n".join(errors), *args1, exc_info=True)
raise HTTPBadGateway("Error on backend, See logs for detail")
if not response.ok:
errors = [
"Error '%s' in response of URL:",
"%s",
"Status: %d",
"Method: %s",
"--- With headers ---",
"%s",
]
args2: List[Union[str, int]] = [
response.reason,
url.url(),
response.status_code,
method,
"\n".join([
"{}: {}".format(
h,
v if h not in ("Authorization", "Cookies") else "***")
for h, v in list(headers.items())
]),
]
if method in ("POST", "PUT") and body is not None:
errors += ["--- Query with body ---", "%s"]
args2.append(body.decode("utf-8"))
errors += ["--- Return content ---", "%s"]
args2.append(response.text)
LOG.error("\n".join(errors), *args2)
raise exception_response(response.status_code)
return response
def test_url_noparam(self):
url = Url("http://example.com/")
url.add_query({})
self.assertEqual(url.url(), "http://example.com/")
def _proxy(
self,
url: Url,
params: Optional[Dict[str, str]] = None,
method: Optional[str] = None,
cache: bool = False,
body: Optional[bytes] = None,
headers: Optional[Dict[str, str]] = None,
) -> requests.models.Response:
# Get query string
params = dict(self.request.params) if params is None else params
url = url.clone().add_query(params, True)
LOG.debug("Send query to URL:\n%s.", url)
if method is None:
method = self.request.method
headers = dict(self.request.headers if headers is None else headers)
# Forward request to target (without Host Header).
# The original Host will be added back by pyramid.
if url.hostname not in self.host_forward_host and "Host" in headers:
headers.pop("Host")
# Forward the request tracking ID to the other service. This will allow to follow the logs belonging
# to a single request coming from the user
headers.setdefault("X-Request-ID", self.request.c2c_request_id)
# If we really want to respect the specification, we should chain with the content of the previous
# proxy, see also:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
forwarded = {
"for": self.request.client_addr,
"proto": self.request.scheme
}
if "Host" in self.request.headers:
forwarded["host"] = self.request.headers["Host"]
forwarded_str = ";".join(["=".join(e) for e in forwarded.items()])
if "Forwarded" in headers:
headers["Forwarded"] = ",".join(
[headers["Forwarded"], forwarded_str])
else:
headers["Forwarded"] = forwarded_str
# Set alternative "X-Forwarded" headers
for forwarded_elements in reversed(headers["Forwarded"].split(",")):
for element in forwarded_elements.split(";"):
key, value = element.split("=")
header_key = f"X-Forwarded-{key.capitalize()}"
header_value = headers.get(header_key)
headers[
header_key] = value if header_value is None else ", ".join(
[header_value, value])
if not cache:
headers["Cache-Control"] = "no-cache"
if method in ("POST", "PUT") and body is None:
body = self.request.body
headers = restrict_headers(headers, self.headers_whitelist,
self.headers_blacklist)
try:
if method in ("POST", "PUT"):
response = requests.request(method,
url.url(),
data=body,
headers=headers,
**self.http_options)
else:
response = requests.request(method,
url.url(),
headers=headers,
**self.http_options)
except Exception:
errors = [
"Error '%s' while getting the URL:", "%s", "Method: %s",
"--- With headers ---", "%s"
]
args1 = [
sys.exc_info()[0],
url,
method,
"\n".join([
f"{h}: {v if h not in ('Authorization', 'Cookies') else '***'}"
for h, v in list(headers.items())
]),
]
if method in ("POST", "PUT") and body is not None:
errors += ["--- Query with body ---", "%s"]
args1.append(body.decode("utf-8"))
LOG.exception("\n".join(errors), *args1)
raise HTTPBadGateway( # pylint: disable=raise-missing-from
"Error on backend, See logs for detail")
if not response.ok:
errors = [
"Error '%s' in response of URL:",
"%s",
"Status: %d",
"Method: %s",
"--- With headers ---",
"%s",
]
args2: List[Union[str, int]] = [
response.reason,
url.url(),
response.status_code,
method,
"\n".join([
f"{h}: {v if h not in ('Authorization', 'Cookies') else '***'}"
for h, v in list(headers.items())
]),
]
if method in ("POST", "PUT") and body is not None:
errors += ["--- Query with body ---", "%s"]
args2.append(body.decode("utf-8"))
errors += ["--- Return content ---", "%s"]
args2.append(response.text)
LOG.error("\n".join(errors), *args2)
raise exception_response(response.status_code)
if not response.headers.get("Content-Type", "").startswith("image/"):
LOG.debug("Get result for URL: %s:\n%s.", url, body)
return response