def test_get_jobs_private_service_public_process_unauthorized_in_query(self):
path = get_path_kvp(jobs_short_uri,
service=self.service_private.name,
process=self.process_public.identifier)
resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
assert resp.status_code == 401
assert resp.content_type == CONTENT_TYPE_APP_JSON
def test_get_jobs_valid_grouping_by_service(self):
path = get_path_kvp(jobs_short_uri, detail="false", groups="service")
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_grouped_info(resp, groups="service")
# ensure that group categories are distinct
for i, grouped_jobs in enumerate(resp.json["groups"]):
categories = grouped_jobs["category"]
for j, grp_jobs in enumerate(resp.json["groups"]):
compared = grp_jobs["category"]
if i == j:
continue
assert categories != compared
# validate groups with expected jobs counts and ids (nb: only public jobs are returned)
if categories["service"] == self.service_public.name:
assert len(grouped_jobs["jobs"]) == 3
assert set(grouped_jobs["jobs"]) == {self.job_info[1].id, self.job_info[5].id, self.job_info[6].id}
elif categories["service"] == self.service_private.name:
assert len(grouped_jobs["jobs"]) == 2
assert set(grouped_jobs["jobs"]) == {self.job_info[7].id, self.job_info[8].id}
elif categories["service"] is None:
assert len(grouped_jobs["jobs"]) == 2
assert set(grouped_jobs["jobs"]) == {self.job_info[0].id, self.job_info[2].id}
else:
pytest.fail("Unknown job grouping 'service' value not expected.")
def test_get_jobs_process_in_query_detail(self):
path = get_path_kvp(jobs_short_uri, process=self.job_info[0].process, detail="true")
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_info(resp)
job_ids = [j["jobID"] for j in resp.json["jobs"]]
assert self.job_info[0].id in job_ids, self.message_with_jobs_mapping("expected in")
assert self.job_info[1].id not in job_ids, self.message_with_jobs_mapping("expected not in")
def test_get_jobs_by_encrypted_email(self):
"""Verifies that literal email can be used as search criterion although not saved in plain text within db."""
email = "*****@*****.**"
body = {
"inputs": [{"id": "test_input", "data": "test"}],
"outputs": [{"id": "test_output", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE}],
"mode": EXECUTE_MODE_ASYNC,
"response": EXECUTE_RESPONSE_DOCUMENT,
"notification_email": email
}
with contextlib.ExitStack() as stack:
for runner in mocked_process_job_runner():
stack.enter_context(runner)
path = "/processes/{}/jobs".format(self.process_public.identifier)
resp = self.app.post_json(path, params=body, headers=self.json_headers)
assert resp.status_code == 201
assert resp.content_type == CONTENT_TYPE_APP_JSON
job_id = resp.json["jobID"]
# verify the email is not in plain text
job = self.job_store.fetch_by_id(job_id)
assert job.notification_email != email and job.notification_email is not None
assert int(job.notification_email, 16) != 0 # email should be encrypted with hex string
path = get_path_kvp(jobs_short_uri, detail="true", notification_email=email)
resp = self.app.get(path, headers=self.json_headers)
assert resp.status_code == 200
assert resp.content_type == CONTENT_TYPE_APP_JSON
assert resp.json["total"] == 1, "Should match exactly 1 email with specified literal string as query param."
assert resp.json["jobs"][0]["jobID"] == job_id
def test_get_jobs_detail_paged(self):
for detail in ("true", 1, "True", "yes"):
path = get_path_kvp(jobs_short_uri, detail=detail)
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_info(resp)
for job in resp.json["jobs"]:
self.check_job_format(job)
def _parse_for_app_req(method, url, **req_kwargs):
"""
WebTest application employs ``params`` instead of ``data``/``json``.
Actual query parameters must be pre-appended to ``url``.
"""
method = method.lower()
url = req_kwargs.pop("base_url", url)
body = req_kwargs.pop("data", None)
query = req_kwargs.pop("query", None)
params = req_kwargs.pop("params", {})
if query:
url += ("" if query.startswith("?") else "?") + query
elif params:
if isinstance(params, str):
url += ("" if params.startswith("?") else "?") + params
else:
url = get_path_kvp(url, **params)
req_kwargs["params"] = body
# remove unsupported parameters that cannot be passed down to TestApp
for key in [
"timeout", "cert", "auth", "ssl_verify", "verify", "language"
]:
req_kwargs.pop(key, None)
req = getattr(app, method)
return url, req, req_kwargs
def test_get_jobs_detail_grouped(self):
for detail in ("true", 1, "True", "yes"):
groups = ["process", "service"]
path = get_path_kvp(jobs_short_uri, detail=detail, groups=groups)
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_grouped_info(resp, groups=groups)
for grouped_jobs in resp.json["groups"]:
for job in grouped_jobs["jobs"]:
self.check_job_format(job)
def test_get_jobs_normal_grouped(self):
for detail in ("false", 0, "False", "no"):
groups = ["process", "service"]
path = get_path_kvp(jobs_short_uri, detail=detail, groups=groups)
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_grouped_info(resp, groups=groups)
for grouped_jobs in resp.json["groups"]:
for job in grouped_jobs["jobs"]:
assert isinstance(job, str)
def test_get_jobs_normal_paged(self):
resp = self.app.get(jobs_short_uri, headers=self.json_headers)
self.check_basic_jobs_info(resp)
for job_id in resp.json["jobs"]:
assert isinstance(job_id, str)
for detail in ("false", 0, "False", "no", "None", "null", None, ""):
path = get_path_kvp(jobs_short_uri, detail=detail)
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_info(resp)
for job_id in resp.json["jobs"]:
assert isinstance(job_id, str)
def test_get_jobs_public_service_no_processes(self):
"""
NOTE:
it is up to the remote service to hide private processes
if the process is invisible, no job should have been executed nor can be fetched
"""
path = get_path_kvp(jobs_short_uri,
service=self.service_public.name,
process=self.process_private.identifier)
with contextlib.ExitStack() as stack:
for job in self.get_job_remote_service_mock([]): # process invisible (not returned by remote)
stack.enter_context(job)
resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
assert resp.status_code == 404
assert resp.content_type == CONTENT_TYPE_APP_JSON
def test_get_jobs_public_service_private_process_unauthorized_in_query(self):
"""
NOTE:
it is up to the remote service to hide private processes
if the process is visible, the a job can be executed and it is automatically considered public
"""
path = get_path_kvp(jobs_short_uri,
service=self.service_public.name,
process=self.process_private.identifier)
with contextlib.ExitStack() as stack:
for runner in self.get_job_remote_service_mock([self.process_private]): # process visible on remote
stack.enter_context(runner)
resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
assert resp.status_code == 200
assert resp.content_type == CONTENT_TYPE_APP_JSON
def _parse_for_app_req(method, url, **req_kwargs):
"""
Obtain request details with adjustments to support specific handling for :class:`webTest.TestApp`.
WebTest application employs ``params`` instead of ``data``/``json``.
Actual query parameters must be pre-appended to ``url``.
"""
method = method.lower()
url = req_kwargs.pop("base_url", url)
body = req_kwargs.pop("data", None)
_json = req_kwargs.pop("json", None)
query = req_kwargs.pop("query", None)
params = req_kwargs.pop("params", {})
if query:
url += ("" if query.startswith("?") else "?") + query
elif params:
if isinstance(params, str):
url += ("" if params.startswith("?") else "?") + params
else:
url = get_path_kvp(url, **params)
req_kwargs["params"] = content = body or _json or {}
# remove unsupported parameters that cannot be passed down to TestApp
for key in [
"timeout", "cert", "auth", "ssl_verify", "verify", "language",
"stream"
]:
req_kwargs.pop(key, None)
cookies = req_kwargs.pop("cookies", None)
if cookies:
cookies = dict(cookies) # in case list of tuples
for name, value in cookies.items():
app.set_cookie(name, value)
# although headers for JSON content can be set, some methods are not working (eg: PUT)
# obtain the corresponding '<method>_json' function to have the proper behaviour
headers = req_kwargs.get("headers", {}) or {}
if ((get_header("Content-Type", headers) == CONTENT_TYPE_APP_JSON
or isinstance(content, (dict, list)))
and hasattr(app, method + "_json")):
method = method + "_json"
if isinstance(content, str):
req_kwargs["params"] = json.loads(req_kwargs["params"])
req = getattr(app, method)
return url, req, req_kwargs
def submit_local_job(request):
# type: (PyramidRequest) -> AnyViewResponse
"""
Execute a process registered locally.
Execution location and method is according to deployed Application Package.
"""
process = get_process(request=request)
ctype = clean_mime_type_format(get_header("content-type", request.headers, default=None), strip_parameters=True)
if ctype in ContentType.ANY_XML:
# Send the XML request to the WPS endpoint which knows how to parse it properly.
# Execution will end up in the same 'submit_job_handler' function as other branch for JSON.
service = get_pywps_service()
wps_params = {"version": "1.0.0", "request": "Execute", "service": "WPS", "identifier": process.id}
request.path_info = get_wps_path(request)
request.query_string = get_path_kvp("", **wps_params)[1:]
location = request.application_url + request.path_info + request.query_string
LOGGER.warning("Route redirection [%s] -> [%s] for WPS-XML support.", request.url, location)
http_request = extend_instance(request, WerkzeugRequest)
http_request.shallow = False
return service.call(http_request)
return submit_job(request, process, tags=["wps-rest"])
def test_get_jobs_public_with_access_and_request_user(self):
"""Verifies that corresponding processes are returned when proper access/user-id are respected."""
uri_direct_jobs = jobs_short_uri
uri_process_jobs = process_jobs_uri.format(process_id=self.process_public.identifier)
uri_provider_jobs = jobs_full_uri.format(
provider_id=self.service_public.name, process_id=self.process_public.identifier)
admin_public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, self.job_info))
admin_private_jobs = list(filter(lambda j: VISIBILITY_PRIVATE in j.access, self.job_info))
editor1_all_jobs = list(filter(lambda j: j.user_id == self.user_editor1_id, self.job_info))
editor1_public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, editor1_all_jobs))
editor1_private_jobs = list(filter(lambda j: VISIBILITY_PRIVATE in j.access, editor1_all_jobs))
public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, self.job_info))
def filter_process(jobs): # type: (Iterable[Job]) -> List[Job]
return list(filter(lambda j: j.process == self.process_public.identifier, jobs))
def filter_service(jobs): # type: (Iterable[Job]) -> List[Job]
return list(filter(lambda j: j.service == self.service_public.name, jobs))
# test variations of [paths, query, user-id, expected-job-ids]
path_jobs_user_req_tests = [
# pylint: disable=C0301,line-too-long,C0326,bad-whitespace
# URI ACCESS USER EXPECTED JOBS
(uri_direct_jobs, None, None, public_jobs), # noqa: E241,E501
(uri_direct_jobs, None, self.user_editor1_id, editor1_all_jobs), # noqa: E241,E501
(uri_direct_jobs, None, self.user_admin_id, self.job_info), # noqa: E241,E501
(uri_direct_jobs, VISIBILITY_PRIVATE, None, public_jobs), # noqa: E241,E501
(uri_direct_jobs, VISIBILITY_PRIVATE, self.user_editor1_id, editor1_private_jobs), # noqa: E241,E501
(uri_direct_jobs, VISIBILITY_PRIVATE, self.user_admin_id, admin_private_jobs), # noqa: E241,E501
(uri_direct_jobs, VISIBILITY_PUBLIC, None, public_jobs), # noqa: E241,E501
(uri_direct_jobs, VISIBILITY_PUBLIC, self.user_editor1_id, editor1_public_jobs), # noqa: E241,E501
(uri_direct_jobs, VISIBILITY_PUBLIC, self.user_admin_id, admin_public_jobs), # noqa: E241,E501
# ---
(uri_process_jobs, None, None, filter_process(public_jobs)), # noqa: E241,E501
(uri_process_jobs, None, self.user_editor1_id, filter_process(editor1_all_jobs)), # noqa: E241,E501
(uri_process_jobs, None, self.user_admin_id, filter_process(self.job_info)), # noqa: E241,E501
(uri_process_jobs, VISIBILITY_PRIVATE, None, filter_process(public_jobs)), # noqa: E241,E501
(uri_process_jobs, VISIBILITY_PRIVATE, self.user_editor1_id, filter_process(editor1_private_jobs)), # noqa: E241,E501
(uri_process_jobs, VISIBILITY_PRIVATE, self.user_admin_id, filter_process(admin_private_jobs)), # noqa: E241,E501
(uri_process_jobs, VISIBILITY_PUBLIC, None, filter_process(public_jobs)), # noqa: E241,E501
(uri_process_jobs, VISIBILITY_PUBLIC, self.user_editor1_id, filter_process(editor1_public_jobs)), # noqa: E241,E501
(uri_process_jobs, VISIBILITY_PUBLIC, self.user_admin_id, filter_process(self.job_info)), # noqa: E241,E501
# ---
(uri_provider_jobs, None, None, filter_service(public_jobs)), # noqa: E241,E501
(uri_provider_jobs, None, self.user_editor1_id, filter_service(editor1_all_jobs)), # noqa: E241,E501
(uri_provider_jobs, None, self.user_admin_id, filter_service(self.job_info)), # noqa: E241,E501
(uri_provider_jobs, VISIBILITY_PRIVATE, None, filter_service(public_jobs)), # noqa: E241,E501
(uri_provider_jobs, VISIBILITY_PRIVATE, self.user_editor1_id, filter_service(editor1_private_jobs)), # noqa: E241,E501
(uri_provider_jobs, VISIBILITY_PRIVATE, self.user_admin_id, filter_service(admin_private_jobs)), # noqa: E241,E501
(uri_provider_jobs, VISIBILITY_PUBLIC, None, filter_service(public_jobs)), # noqa: E241,E501
(uri_provider_jobs, VISIBILITY_PUBLIC, self.user_editor1_id, filter_service(editor1_public_jobs)), # noqa: E241,E501
(uri_provider_jobs, VISIBILITY_PUBLIC, self.user_admin_id, filter_service(self.job_info)), # noqa: E241,E501
] # type: List[Tuple[str, str, Union[None, int], List[Job]]]
for i, (path, access, user_id, expected_jobs) in enumerate(path_jobs_user_req_tests):
with contextlib.ExitStack() as stack:
for patch in self.get_job_request_auth_mock(user_id):
stack.enter_context(patch)
for patch in self.get_job_remote_service_mock([self.process_public]):
stack.enter_context(patch)
test = get_path_kvp(path, access=access) if access else path
resp = self.app.get(test, headers=self.json_headers)
self.check_basic_jobs_info(resp)
job_ids = [job.id for job in expected_jobs]
job_match = all(job in job_ids for job in resp.json["jobs"])
test_values = dict(path=path, access=access, user_id=user_id)
assert job_match, self.message_with_jobs_diffs(resp.json["jobs"], job_ids, test_values, index=i)
def test_get_path_kvp():
res = get_path_kvp("http://localhost",
test1="value1",
test2=["sub1", "sub2"])
assert res == "http://localhost?test1=value1&test2=sub1,sub2"
def test_get_jobs_process_in_query_normal(self):
path = get_path_kvp(jobs_short_uri, process=self.job_info[0].process)
resp = self.app.get(path, headers=self.json_headers)
self.check_basic_jobs_info(resp)
assert self.job_info[0].id in resp.json["jobs"], self.message_with_jobs_mapping("expected in")
assert self.job_info[1].id not in resp.json["jobs"], self.message_with_jobs_mapping("expected not in")
def get_process_list_links(request, paging, total, provider=None):
# type: (Request, Dict[str, int], Optional[int], Optional[Service]) -> List[JSON]
"""
Obtains a list of all relevant links for the corresponding :term:`Process` listing defined by query parameters.
:raises IndexError: if the paging values are out of bounds compared to available total :term:`Process`.
"""
# reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.)
kvp_params = {
param: value
for param, value in request.params.items() if param != "page"
}
base_url = get_weaver_url(request)
links = []
if provider:
proc_path = sd.provider_processes_service.path.format(
provider_id=provider.id)
links.extend(provider.links(request, self_link="provider"))
else:
proc_path = sd.processes_service.path
proc_url = base_url + proc_path
links.extend([
{
"href": proc_url,
"rel": "collection",
"type": CONTENT_TYPE_APP_JSON,
"title": "Process listing (no filtering queries applied)."
},
{
"href": proc_url,
"rel": "search",
"type": CONTENT_TYPE_APP_JSON,
"title": "Generic query endpoint to list processes."
},
{
"href": proc_url + "?detail=false",
"rel": "preview",
"type": CONTENT_TYPE_APP_JSON,
"title": "Process listing summary (identifiers and count only)."
},
{
"href": proc_url,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/processes",
"type": CONTENT_TYPE_APP_JSON,
"title": "List of registered local processes."
},
{
"href": get_path_kvp(proc_url, **request.params),
"rel": "self",
"type": CONTENT_TYPE_APP_JSON,
"title": "Current process listing."
},
])
if provider:
prov_url = proc_url.rsplit("/", 1)[0]
links.append({
"href": prov_url,
"rel": "up",
"type": CONTENT_TYPE_APP_JSON,
"title": "Provider description."
})
else:
links.append({
"href": base_url,
"rel": "up",
"type": CONTENT_TYPE_APP_JSON,
"title": "API entrypoint."
})
cur_page = paging.get("page", None)
per_page = paging.get("limit", None)
if all(isinstance(num, int) for num in [cur_page, per_page, total]):
max_page = max(math.ceil(total / per_page) - 1, 0)
if cur_page < 0 or cur_page > max_page:
raise IndexError(
f"Page index {cur_page} is out of range from [0,{max_page}].")
links.extend([
{
"href": get_path_kvp(proc_url, page=cur_page, **kvp_params),
"rel": "current",
"type": CONTENT_TYPE_APP_JSON,
"title": "Current page of processes query listing."
},
{
"href": get_path_kvp(proc_url, page=0, **kvp_params),
"rel": "first",
"type": CONTENT_TYPE_APP_JSON,
"title": "First page of processes query listing."
},
{
"href": get_path_kvp(proc_url, page=max_page, **kvp_params),
"rel": "last",
"type": CONTENT_TYPE_APP_JSON,
"title": "Last page of processes query listing."
},
])
if cur_page > 0:
links.append({
"href":
get_path_kvp(proc_url, page=cur_page - 1, **kvp_params),
"rel":
"prev",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Previous page of processes query listing."
})
if cur_page < max_page:
links.append({
"href":
get_path_kvp(proc_url, page=cur_page + 1, **kvp_params),
"rel":
"next",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Next page of processes query listing."
})
return links
def test_get_jobs_private_process_not_returned_in_query(self):
path = get_path_kvp(jobs_short_uri, process=self.process_private.identifier)
resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
assert resp.status_code == 401
assert resp.content_type == CONTENT_TYPE_APP_JSON
def get_job_list_links(job_total, filters, request):
# type: (int, Dict[str, AnyValueType], Request) -> List[JSON]
"""
Obtains a list of all relevant links for the corresponding job listing defined by query parameter filters.
:raises IndexError: if the paging values are out of bounds compared to available total :term:`Job` matching search.
"""
base_url = get_weaver_url(request)
# reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.)
kvp_params = {
param: value
for param, value in request.params.items() if param != "page"
}
# patch datetime that have some extra character manipulation (reapply '+' auto-converted to ' ' by params parser)
if "datetime" in kvp_params:
kvp_params["datetime"] = kvp_params["datetime"].replace(" ", "+")
alt_kvp = deepcopy(kvp_params)
# request job uses general endpoint, obtain the full path if any service/process was given as alternate location
if request.path.startswith(sd.jobs_service.path):
job_path = base_url + sd.jobs_service.path
alt_path = None
parent_url = None
# cannot generate full path apply for 'service' by itself
if filters["process"] and filters["service"]:
alt_path = base_url + sd.provider_jobs_service.path.format(
provider_id=filters["service"], process_id=filters["process"])
parent_url = alt_path.rsplit("/", 1)[0]
elif filters["process"]:
alt_path = base_url + sd.process_jobs_service.path.format(
process_id=filters["process"])
parent_url = alt_path.rsplit("/", 1)[0]
for param in ["service", "provider", "process"]:
alt_kvp.pop(param, None)
# path is whichever specific service/process endpoint, jobs are pre-filtered by them
# transform sub-endpoints into matching query parameters and use generic path as alternate location
else:
job_path = base_url + request.path
alt_path = base_url + sd.jobs_service.path
alt_kvp["process"] = filters["process"]
if filters["service"]:
alt_kvp["provider"] = filters["service"]
parent_url = job_path.rsplit("/", 1)[0]
cur_page = filters["page"]
per_page = filters["limit"]
max_page = max(math.ceil(job_total / per_page) - 1, 0)
if cur_page < 0 or cur_page > max_page:
raise IndexError(
f"Page index {cur_page} is out of range from [0,{max_page}].")
alt_links = []
if alt_path:
alt_links = [{
"href":
get_path_kvp(alt_path, page=cur_page, **alt_kvp),
"rel":
"alternate",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Alternate endpoint with equivalent set of filtered jobs."
}]
links = alt_links + [
{
"href": job_path,
"rel": "collection",
"type": CONTENT_TYPE_APP_JSON,
"title": "Complete job listing (no filtering queries applied)."
},
{
"href": base_url + sd.jobs_service.path,
"rel": "search",
"type": CONTENT_TYPE_APP_JSON,
"title": "Generic query endpoint to search for jobs."
},
{
"href": job_path + "?detail=false",
"rel": "preview",
"type": CONTENT_TYPE_APP_JSON,
"title": "Job listing summary (UUID and count only)."
},
{
"href": job_path,
"rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list",
"type": CONTENT_TYPE_APP_JSON,
"title": "List of registered jobs."
},
{
"href": get_path_kvp(job_path, page=cur_page, **kvp_params),
"rel": "current",
"type": CONTENT_TYPE_APP_JSON,
"title": "Current page of job query listing."
},
{
"href": get_path_kvp(job_path, page=0, **kvp_params),
"rel": "first",
"type": CONTENT_TYPE_APP_JSON,
"title": "First page of job query listing."
},
{
"href": get_path_kvp(job_path, page=max_page, **kvp_params),
"rel": "last",
"type": CONTENT_TYPE_APP_JSON,
"title": "Last page of job query listing."
},
]
if cur_page > 0:
links.append({
"href":
get_path_kvp(job_path, page=cur_page - 1, **kvp_params),
"rel":
"prev",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Previous page of job query listing."
})
if cur_page < max_page:
links.append({
"href":
get_path_kvp(job_path, page=cur_page + 1, **kvp_params),
"rel":
"next",
"type":
CONTENT_TYPE_APP_JSON,
"title":
"Next page of job query listing."
})
if parent_url:
links.append({
"href": parent_url,
"rel": "up",
"type": CONTENT_TYPE_APP_JSON,
"title": "Parent collection for which listed jobs apply."
})
return links
def test_get_jobs_service_and_process_unknown_in_query(self):
path = get_path_kvp(jobs_short_uri, service="unknown-service-id", process="unknown-process-id")
resp = self.app.get(path, headers=self.json_headers, expect_errors=True)
assert resp.status_code == 404
assert resp.content_type == CONTENT_TYPE_APP_JSON